").attr({role:"log","aria-live":"assertive","aria-relevant":"additions"}).appendTo(this.document[0].body),this._addClass(this.liveRegion,null,"ui-helper-hidden-accessible"),this.disabledTitles=t([])},_setOption:function(e,i){var s=this;this._super(e,i),"content"===e&&t.each(this.tooltips,function(t,e){s._updateContent(e.element)})},_setOptionDisabled:function(t){this[t?"_disable":"_enable"]()},_disable:function(){var e=this;t.each(this.tooltips,function(i,s){var n=t.Event("blur");n.target=n.currentTarget=s.element[0],e.close(n,!0)}),this.disabledTitles=this.disabledTitles.add(this.element.find(this.options.items).addBack().filter(function(){var e=t(this);return e.is("[title]")?e.data("ui-tooltip-title",e.attr("title")).removeAttr("title"):void 0}))},_enable:function(){this.disabledTitles.each(function(){var e=t(this);e.data("ui-tooltip-title")&&e.attr("title",e.data("ui-tooltip-title"))}),this.disabledTitles=t([])},open:function(e){var i=this,s=t(e?e.target:this.element).closest(this.options.items);s.length&&!s.data("ui-tooltip-id")&&(s.attr("title")&&s.data("ui-tooltip-title",s.attr("title")),s.data("ui-tooltip-open",!0),e&&"mouseover"===e.type&&s.parents().each(function(){var e,s=t(this);s.data("ui-tooltip-open")&&(e=t.Event("blur"),e.target=e.currentTarget=this,i.close(e,!0)),s.attr("title")&&(s.uniqueId(),i.parents[this.id]={element:this,title:s.attr("title")},s.attr("title",""))}),this._registerCloseHandlers(e,s),this._updateContent(s,e))},_updateContent:function(t,e){var i,s=this.options.content,n=this,o=e?e.type:null;return"string"==typeof s||s.nodeType||s.jquery?this._open(e,t,s):(i=s.call(t[0],function(i){n._delay(function(){t.data("ui-tooltip-open")&&(e&&(e.type=o),this._open(e,t,i))})}),i&&this._open(e,t,i),void 0)},_open:function(e,i,s){function n(t){l.of=t,a.is(":hidden")||a.position(l)}var o,a,r,h,l=t.extend({},this.options.position);if(s){if(o=this._find(i))return o.tooltip.find(".ui-tooltip-content").html(s),void 0;i.is("[title]")&&(e&&"mouseover"===e.type?i.attr("title",""):i.removeAttr("title")),o=this._tooltip(i),a=o.tooltip,this._addDescribedBy(i,a.attr("id")),a.find(".ui-tooltip-content").html(s),this.liveRegion.children().hide(),h=t("
").html(a.find(".ui-tooltip-content").html()),h.removeAttr("name").find("[name]").removeAttr("name"),h.removeAttr("id").find("[id]").removeAttr("id"),h.appendTo(this.liveRegion),this.options.track&&e&&/^mouse/.test(e.type)?(this._on(this.document,{mousemove:n}),n(e)):a.position(t.extend({of:i},this.options.position)),a.hide(),this._show(a,this.options.show),this.options.track&&this.options.show&&this.options.show.delay&&(r=this.delayedShow=setInterval(function(){a.is(":visible")&&(n(l.of),clearInterval(r))},t.fx.interval)),this._trigger("open",e,{tooltip:a})}},_registerCloseHandlers:function(e,i){var s={keyup:function(e){if(e.keyCode===t.ui.keyCode.ESCAPE){var s=t.Event(e);s.currentTarget=i[0],this.close(s,!0)}}};i[0]!==this.element[0]&&(s.remove=function(){this._removeTooltip(this._find(i).tooltip)}),e&&"mouseover"!==e.type||(s.mouseleave="close"),e&&"focusin"!==e.type||(s.focusout="close"),this._on(!0,i,s)},close:function(e){var i,s=this,n=t(e?e.currentTarget:this.element),o=this._find(n);return o?(i=o.tooltip,o.closing||(clearInterval(this.delayedShow),n.data("ui-tooltip-title")&&!n.attr("title")&&n.attr("title",n.data("ui-tooltip-title")),this._removeDescribedBy(n),o.hiding=!0,i.stop(!0),this._hide(i,this.options.hide,function(){s._removeTooltip(t(this))}),n.removeData("ui-tooltip-open"),this._off(n,"mouseleave focusout keyup"),n[0]!==this.element[0]&&this._off(n,"remove"),this._off(this.document,"mousemove"),e&&"mouseleave"===e.type&&t.each(this.parents,function(e,i){t(i.element).attr("title",i.title),delete s.parents[e]}),o.closing=!0,this._trigger("close",e,{tooltip:i}),o.hiding||(o.closing=!1)),void 0):(n.removeData("ui-tooltip-open"),void 0)},_tooltip:function(e){var i=t("
").attr("role","tooltip"),s=t("
").appendTo(i),n=i.uniqueId().attr("id");return this._addClass(s,"ui-tooltip-content"),this._addClass(i,"ui-tooltip","ui-widget ui-widget-content"),i.appendTo(this._appendTo(e)),this.tooltips[n]={element:e,tooltip:i}},_find:function(t){var e=t.data("ui-tooltip-id");return e?this.tooltips[e]:null},_removeTooltip:function(t){t.remove(),delete this.tooltips[t.attr("id")]},_appendTo:function(t){var e=t.closest(".ui-front, dialog");return e.length||(e=this.document[0].body),e},_destroy:function(){var e=this;t.each(this.tooltips,function(i,s){var n=t.Event("blur"),o=s.element;n.target=n.currentTarget=o[0],e.close(n,!0),t("#"+i).remove(),o.data("ui-tooltip-title")&&(o.attr("title")||o.attr("title",o.data("ui-tooltip-title")),o.removeData("ui-tooltip-title"))}),this.liveRegion.remove()}}),t.uiBackCompat!==!1&&t.widget("ui.tooltip",t.ui.tooltip,{options:{tooltipClass:null},_tooltip:function(){var t=this._superApply(arguments);return this.options.tooltipClass&&t.tooltip.addClass(this.options.tooltipClass),t}}),t.ui.tooltip});
\ No newline at end of file
diff --git a/docs/waifu_plugin/source b/docs/waifu_plugin/source
index 82d2ed01245d9ab6a6d70576398b96286b24790d..7201fd631355230715b825c8046718da50c2c75c 100644
--- a/docs/waifu_plugin/source
+++ b/docs/waifu_plugin/source
@@ -1 +1 @@
-https://github.com/fghrsh/live2d_demo
+https://github.com/fghrsh/live2d_demo
\ No newline at end of file
diff --git a/docs/waifu_plugin/waifu-tips.js b/docs/waifu_plugin/waifu-tips.js
index 6ae98395a14239861fbc8250d190971e34590434..8f9533a19e7d4914bde888ee2a107e4430242968 100644
--- a/docs/waifu_plugin/waifu-tips.js
+++ b/docs/waifu_plugin/waifu-tips.js
@@ -5,11 +5,11 @@ window.live2d_settings = Array(); /*
／｀ｰ' L/／｀ヽ､ Live2D 看板娘 参数设置
/ ／, /| , , ', Version 1.4.2
ｲ / /-‐/ ｉ L_ ﾊ ヽ! i Update 2018.11.12
- ﾚ ﾍ 7ｲ｀ﾄ ﾚ'ｧ-ﾄ､!ハ| |
+ ﾚ ﾍ 7ｲ｀ﾄ ﾚ'ｧ-ﾄ､!ハ| |
!,/7 '0' ´0iソ| |
|.从" _ ,,,, / |./ | 网页添加 Live2D 看板娘
ﾚ'| i＞.､,,__ _,.イ / .i | https://www.fghrsh.net/post/123.html
- ﾚ'| | / k_７_/ﾚ'ヽ, ﾊ. |
+ ﾚ'| | / k_７_/ﾚ'ヽ, ﾊ. |
| |/i 〈|/ i ,.ﾍ | i | Thanks
.|/ / ｉ： ﾍ! ＼ | journey-ad / https://github.com/journey-ad/live2d_src
kヽ>､ﾊ _,.ﾍ､ /､! xiazeyu / https://github.com/xiazeyu/live2d-widget.js
@@ -77,11 +77,11 @@ String.prototype.render = function(context) {
return this.replace(tokenReg, function (word, slash1, token, slash2) {
if (slash1 || slash2) { return word.replace('\\', ''); }
-
+
var variables = token.replace(/\s/g, '').split('.');
var currentObject = context;
var i, length, variable;
-
+
for (i = 0, length = variables.length; i < length; ++i) {
variable = variables[i];
currentObject = currentObject[variable];
@@ -101,9 +101,9 @@ function showMessage(text, timeout, flag) {
if(flag || sessionStorage.getItem('waifu-text') === '' || sessionStorage.getItem('waifu-text') === null){
if(Array.isArray(text)) text = text[Math.floor(Math.random() * text.length + 1)-1];
if (live2d_settings.showF12Message) console.log('[Message]', text.replace(/<[^<>]+>/g,''));
-
+
if(flag) sessionStorage.setItem('waifu-text', text);
-
+
$('.waifu-tips').stop();
$('.waifu-tips').html(text).fadeTo(200, 1);
if (timeout === undefined) timeout = 5000;
@@ -121,15 +121,15 @@ function hideMessage(timeout) {
function initModel(waifuPath, type) {
/* console welcome message */
eval(function(p,a,c,k,e,r){e=function(c){return(c
35?String.fromCharCode(c+29):c.toString(36))};if(!''.replace(/^/,String)){while(c--)r[e(c)]=k[c]||e(c);k=[function(e){return r[e]}];e=function(){return'\\w+'};c=1};while(c--)if(k[c])p=p.replace(new RegExp('\\b'+e(c)+'\\b','g'),k[c]);return p}('8.d(" ");8.d("\\U,.\\y\\5.\\1\\1\\1\\1/\\1,\\u\\2 \\H\\n\\1\\1\\1\\1\\1\\b \', !-\\r\\j-i\\1/\\1/\\g\\n\\1\\1\\1 \\1 \\a\\4\\f\'\\1\\1\\1 L/\\a\\4\\5\\2\\n\\1\\1 \\1 /\\1 \\a,\\1 /|\\1 ,\\1 ,\\1\\1\\1 \',\\n\\1\\1\\1\\q \\1/ /-\\j/\\1\\h\\E \\9 \\5!\\1 i\\n\\1\\1\\1 \\3 \\6 7\\q\\4\\c\\1 \\3\'\\s-\\c\\2!\\t|\\1 |\\n\\1\\1\\1\\1 !,/7 \'0\'\\1\\1 \\X\\w| \\1 |\\1\\1\\1\\n\\1\\1\\1\\1 |.\\x\\"\\1\\l\\1\\1 ,,,, / |./ \\1 |\\n\\1\\1\\1\\1 \\3\'| i\\z.\\2,,A\\l,.\\B / \\1.i \\1|\\n\\1\\1\\1\\1\\1 \\3\'| | / C\\D/\\3\'\\5,\\1\\9.\\1|\\n\\1\\1\\1\\1\\1\\1 | |/i \\m|/\\1 i\\1,.\\6 |\\F\\1|\\n\\1\\1\\1\\1\\1\\1.|/ /\\1\\h\\G \\1 \\6!\\1\\1\\b\\1|\\n\\1\\1\\1 \\1 \\1 k\\5>\\2\\9 \\1 o,.\\6\\2 \\1 /\\2!\\n\\1\\1\\1\\1\\1\\1 !\'\\m//\\4\\I\\g\', \\b \\4\'7\'\\J\'\\n\\1\\1\\1\\1\\1\\1 \\3\'\\K|M,p,\\O\\3|\\P\\n\\1\\1\\1\\1\\1 \\1\\1\\1\\c-,/\\1|p./\\n\\1\\1\\1\\1\\1 \\1\\1\\1\'\\f\'\\1\\1!o,.:\\Q \\R\\S\\T v"+e.V+" / W "+e.N);8.d(" ");',60,60,'|u3000|uff64|uff9a|uff40|u30fd|uff8d||console|uff8a|uff0f|uff3c|uff84|log|live2d_settings|uff70|u00b4|uff49||u2010||u3000_|u3008||_|___|uff72|u2500|uff67|u30cf|u30fc||u30bd|u4ece|u30d8|uff1e|__|u30a4|k_|uff17_|u3000L_|u3000i|uff1a|u3009|uff34|uff70r|u30fdL__||___i|l2dVerDate|u30f3|u30ce|nLive2D|u770b|u677f|u5a18|u304f__|l2dVersion|FGHRSH|u00b40i'.split('|'),0,{}));
-
+
/* 判断 JQuery */
if (typeof($.ajax) != 'function') typeof(jQuery.ajax) == 'function' ? window.$ = jQuery : console.log('[Error] JQuery is not defined.');
-
+
/* 加载看板娘样式 */
live2d_settings.waifuSize = live2d_settings.waifuSize.split('x');
live2d_settings.waifuTipsSize = live2d_settings.waifuTipsSize.split('x');
live2d_settings.waifuEdgeSide = live2d_settings.waifuEdgeSide.split(':');
-
+
$("#live2d").attr("width",live2d_settings.waifuSize[0]);
$("#live2d").attr("height",live2d_settings.waifuSize[1]);
$(".waifu-tips").width(live2d_settings.waifuTipsSize[0]);
@@ -138,32 +138,32 @@ function initModel(waifuPath, type) {
$(".waifu-tips").css("font-size",live2d_settings.waifuFontSize);
$(".waifu-tool").css("font-size",live2d_settings.waifuToolFont);
$(".waifu-tool span").css("line-height",live2d_settings.waifuToolLine);
-
+
if (live2d_settings.waifuEdgeSide[0] == 'left') $(".waifu").css("left",live2d_settings.waifuEdgeSide[1]+'px');
else if (live2d_settings.waifuEdgeSide[0] == 'right') $(".waifu").css("right",live2d_settings.waifuEdgeSide[1]+'px');
-
+
window.waifuResize = function() { $(window).width() <= Number(live2d_settings.waifuMinWidth.replace('px','')) ? $(".waifu").hide() : $(".waifu").show(); };
if (live2d_settings.waifuMinWidth != 'disable') { waifuResize(); $(window).resize(function() {waifuResize()}); }
-
+
try {
if (live2d_settings.waifuDraggable == 'axis-x') $(".waifu").draggable({ axis: "x", revert: live2d_settings.waifuDraggableRevert });
else if (live2d_settings.waifuDraggable == 'unlimited') $(".waifu").draggable({ revert: live2d_settings.waifuDraggableRevert });
else $(".waifu").css("transition", 'all .3s ease-in-out');
} catch(err) { console.log('[Error] JQuery UI is not defined.') }
-
+
live2d_settings.homePageUrl = live2d_settings.homePageUrl == 'auto' ? window.location.protocol+'//'+window.location.hostname+'/' : live2d_settings.homePageUrl;
if (window.location.protocol == 'file:' && live2d_settings.modelAPI.substr(0,2) == '//') live2d_settings.modelAPI = 'http:'+live2d_settings.modelAPI;
-
+
$('.waifu-tool .fui-home').click(function (){
//window.location = 'https://www.fghrsh.net/';
window.location = live2d_settings.homePageUrl;
});
-
+
$('.waifu-tool .fui-info-circle').click(function (){
//window.open('https://imjad.cn/archives/lab/add-dynamic-poster-girl-with-live2d-to-your-blog-02');
window.open(live2d_settings.aboutPageUrl);
});
-
+
if (typeof(waifuPath) == "object") loadTipsMessage(waifuPath); else {
$.ajax({
cache: true,
@@ -172,7 +172,7 @@ function initModel(waifuPath, type) {
success: function (result){ loadTipsMessage(result); }
});
}
-
+
if (!live2d_settings.showToolMenu) $('.waifu-tool').hide();
if (!live2d_settings.canCloseLive2d) $('.waifu-tool .fui-cross').hide();
if (!live2d_settings.canSwitchModel) $('.waifu-tool .fui-eye').hide();
@@ -185,7 +185,7 @@ function initModel(waifuPath, type) {
if (waifuPath === undefined) waifuPath = '';
var modelId = localStorage.getItem('modelId');
var modelTexturesId = localStorage.getItem('modelTexturesId');
-
+
if (!live2d_settings.modelStorage || modelId == null) {
var modelId = live2d_settings.modelId;
var modelTexturesId = live2d_settings.modelTexturesId;
@@ -204,7 +204,7 @@ function loadModel(modelId, modelTexturesId=0) {
function loadTipsMessage(result) {
window.waifu_tips = result;
-
+
$.each(result.mouseover, function (index, tips){
$(document).on("mouseover", tips.selector, function (){
var text = getRandText(tips.text);
@@ -223,50 +223,82 @@ function loadTipsMessage(result) {
var now = new Date();
var after = tips.date.split('-')[0];
var before = tips.date.split('-')[1] || after;
-
- if((after.split('/')[0] <= now.getMonth()+1 && now.getMonth()+1 <= before.split('/')[0]) &&
+
+ if((after.split('/')[0] <= now.getMonth()+1 && now.getMonth()+1 <= before.split('/')[0]) &&
(after.split('/')[1] <= now.getDate() && now.getDate() <= before.split('/')[1])){
var text = getRandText(tips.text);
text = text.render({year: now.getFullYear()});
showMessage(text, 6000, true);
}
});
-
+
if (live2d_settings.showF12OpenMsg) {
re.toString = function() {
showMessage(getRandText(result.waifu.console_open_msg), 5000, true);
return '';
};
}
-
+
if (live2d_settings.showCopyMessage) {
$(document).on('copy', function() {
showMessage(getRandText(result.waifu.copy_message), 5000, true);
});
}
-
+
$('.waifu-tool .fui-photo').click(function(){
showMessage(getRandText(result.waifu.screenshot_message), 5000, true);
window.Live2D.captureName = live2d_settings.screenshotCaptureName;
window.Live2D.captureFrame = true;
});
-
+
$('.waifu-tool .fui-cross').click(function(){
sessionStorage.setItem('waifu-dsiplay', 'none');
showMessage(getRandText(result.waifu.hidden_message), 1300, true);
window.setTimeout(function() {$('.waifu').hide();}, 1300);
});
-
+
window.showWelcomeMessage = function(result) {
- showMessage('欢迎使用GPT-Academic', 6000);
+ var text;
+ if (window.location.href == live2d_settings.homePageUrl) {
+ var now = (new Date()).getHours();
+ if (now > 23 || now <= 5) text = getRandText(result.waifu.hour_tips['t23-5']);
+ else if (now > 5 && now <= 7) text = getRandText(result.waifu.hour_tips['t5-7']);
+ else if (now > 7 && now <= 11) text = getRandText(result.waifu.hour_tips['t7-11']);
+ else if (now > 11 && now <= 14) text = getRandText(result.waifu.hour_tips['t11-14']);
+ else if (now > 14 && now <= 17) text = getRandText(result.waifu.hour_tips['t14-17']);
+ else if (now > 17 && now <= 19) text = getRandText(result.waifu.hour_tips['t17-19']);
+ else if (now > 19 && now <= 21) text = getRandText(result.waifu.hour_tips['t19-21']);
+ else if (now > 21 && now <= 23) text = getRandText(result.waifu.hour_tips['t21-23']);
+ else text = getRandText(result.waifu.hour_tips.default);
+ } else {
+ var referrer_message = result.waifu.referrer_message;
+ if (document.referrer !== '') {
+ var referrer = document.createElement('a');
+ referrer.href = document.referrer;
+ var domain = referrer.hostname.split('.')[1];
+ if (window.location.hostname == referrer.hostname)
+ text = referrer_message.localhost[0] + document.title.split(referrer_message.localhost[2])[0] + referrer_message.localhost[1];
+ else if (domain == 'baidu')
+ text = referrer_message.baidu[0] + referrer.search.split('&wd=')[1].split('&')[0] + referrer_message.baidu[1];
+ else if (domain == 'so')
+ text = referrer_message.so[0] + referrer.search.split('&q=')[1].split('&')[0] + referrer_message.so[1];
+ else if (domain == 'google')
+ text = referrer_message.google[0] + document.title.split(referrer_message.google[2])[0] + referrer_message.google[1];
+ else {
+ $.each(result.waifu.referrer_hostname, function(i,val) {if (i==referrer.hostname) referrer.hostname = getRandText(val)});
+ text = referrer_message.default[0] + referrer.hostname + referrer_message.default[1];
+ }
+ } else text = referrer_message.none[0] + document.title.split(referrer_message.none[2])[0] + referrer_message.none[1];
+ }
+ showMessage(text, 6000);
}; if (live2d_settings.showWelcomeMessage) showWelcomeMessage(result);
-
+
var waifu_tips = result.waifu;
-
+
function loadOtherModel() {
var modelId = modelStorageGetItem('modelId');
var modelRandMode = live2d_settings.modelRandMode;
-
+
$.ajax({
cache: modelRandMode == 'switch' ? true : false,
url: live2d_settings.modelAPI+modelRandMode+'/?id='+modelId,
@@ -279,12 +311,12 @@ function loadTipsMessage(result) {
}
});
}
-
+
function loadRandTextures() {
var modelId = modelStorageGetItem('modelId');
var modelTexturesId = modelStorageGetItem('modelTexturesId');
var modelTexturesRandMode = live2d_settings.modelTexturesRandMode;
-
+
$.ajax({
cache: modelTexturesRandMode == 'switch' ? true : false,
url: live2d_settings.modelAPI+modelTexturesRandMode+'_textures/?id='+modelId+'-'+modelTexturesId,
@@ -297,32 +329,32 @@ function loadTipsMessage(result) {
}
});
}
-
+
function modelStorageGetItem(key) { return live2d_settings.modelStorage ? localStorage.getItem(key) : sessionStorage.getItem(key); }
-
+
/* 检测用户活动状态，并在空闲时显示一言 */
if (live2d_settings.showHitokoto) {
window.getActed = false; window.hitokotoTimer = 0; window.hitokotoInterval = false;
$(document).mousemove(function(e){getActed = true;}).keydown(function(){getActed = true;});
setInterval(function(){ if (!getActed) ifActed(); else elseActed(); }, 1000);
}
-
+
function ifActed() {
if (!hitokotoInterval) {
hitokotoInterval = true;
hitokotoTimer = window.setInterval(showHitokotoActed, 30000);
}
}
-
+
function elseActed() {
getActed = hitokotoInterval = false;
window.clearInterval(hitokotoTimer);
}
-
+
function showHitokotoActed() {
if ($(document)[0].visibilityState == 'visible') showHitokoto();
}
-
+
function showHitokoto() {
switch(live2d_settings.hitokotoAPI) {
case 'lwl12.com':
@@ -366,7 +398,7 @@ function loadTipsMessage(result) {
});
}
}
-
+
$('.waifu-tool .fui-eye').click(function (){loadOtherModel()});
$('.waifu-tool .fui-user').click(function (){loadRandTextures()});
$('.waifu-tool .fui-chat').click(function (){showHitokoto()});
diff --git a/docs/waifu_plugin/waifu-tips.json b/docs/waifu_plugin/waifu-tips.json
index c7d84e3e6835a3d0926c3a4539bd223affd0265c..229d5a14cae722e7f7dbaf23e23d11e6ce233199 100644
--- a/docs/waifu_plugin/waifu-tips.json
+++ b/docs/waifu_plugin/waifu-tips.json
@@ -31,7 +31,7 @@
},
"model_message": {
"1": ["来自 Potion Maker 的 Pio 酱 ~"],
- "2": ["来自 Potion Maker 的 Tia 酱 ~"]
+ "2": ["来自 Potion Maker 的 Tia 酱 ~"]
},
"hitokoto_api_message": {
"lwl12.com": ["这句一言来自 『{source}』", "，是 {creator} 投稿的", "。"],
@@ -83,8 +83,8 @@
"很多强大的函数插件隐藏在下拉菜单中呢。",
"红色的插件，使用之前需要把文件上传进去哦。",
"想添加功能按钮吗？读读readme很容易就学会啦。",
- "敏感或机密的信息，不可以问AI的哦！",
- "LLM究竟是划时代的创新，还是扼杀创造力的毒药呢？"
+ "敏感或机密的信息，不可以问chatGPT的哦！",
+ "chatGPT究竟是划时代的创新，还是扼杀创造力的毒药呢？"
] }
],
"click": [
@@ -92,6 +92,8 @@
"selector": ".waifu #live2d",
"text": [
"是…是不小心碰到了吧",
+ "萝莉控是什么呀",
+ "你看到我的小熊了吗",
"再摸的话我可要报警了！⌇●﹏●⌇",
"110吗，这里有个变态一直在摸我(ó﹏ò｡)"
]
@@ -111,4 +113,4 @@
{ "date": "11/05-11/12", "text": ["今年的双十一是和谁一起过的呢~"] },
{ "date": "12/20-12/31", "text": ["这几天是圣诞节，主人肯定又去剁手买买买了~"] }
]
-}
+}
\ No newline at end of file
diff --git a/docs/waifu_plugin/waifu.css b/docs/waifu_plugin/waifu.css
index 0a50344d79a66972067476f308a1544968e00a40..42639df0794e46fc58f66e2c772e2bf9ba605eed 100644
--- a/docs/waifu_plugin/waifu.css
+++ b/docs/waifu_plugin/waifu.css
@@ -287,4 +287,4 @@
}
.fui-user:before {
content: "\e631";
-}
+}
\ No newline at end of file
diff --git a/flagged/modeling_moss.py b/flagged/modeling_moss.py
deleted file mode 100644
index f191bf532c0e1b769e773f932a35f099f782de44..0000000000000000000000000000000000000000
--- a/flagged/modeling_moss.py
+++ /dev/null
@@ -1,2952 +0,0 @@
-""" PyTorch Moss model."""
-
-from typing import Optional, Tuple, Union
-
-import torch
-import torch.utils.checkpoint
-from torch import nn
-from torch.nn import CrossEntropyLoss
-import transformers
-from transformers.activations import ACT2FN
-from transformers.modeling_utils import PreTrainedModel
-from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
-from transformers.utils import (
- add_code_sample_docstrings,
- add_start_docstrings,
- add_start_docstrings_to_model_forward,
- logging
-)
-
-from .configuration_moss import MossConfig
-
-logger = logging.get_logger(__name__)
-
-_CHECKPOINT_FOR_DOC = "fnlp/moss-moon-003-base"
-_CONFIG_FOR_DOC = "MossConfig"
-
-
-MOSS_PRETRAINED_MODEL_ARCHIVE_LIST = [
- "fnlp/moss-moon-003-base",
- "fnlp/moss-moon-003-sft",
- "fnlp/moss-moon-003-sft-plugin",
- "fnlp/moss-moon-003-sft-int4",
- "fnlp/moss-moon-003-sft-plugin-int4",
- "fnlp/moss-moon-003-sft-int8",
- "fnlp/moss-moon-003-sft-plugin-int8",
-]
-
-
-# Copied from transformers.models.gptj.modeling_gptj.create_sinusoidal_positions
-def create_sinusoidal_positions(num_pos: int, dim: int) -> torch.Tensor:
- inv_freq = 1.0 / (10000 ** (torch.arange(0, dim, 2) / dim))
- sinusoid_inp = torch.einsum("i , j -> i j", torch.arange(num_pos, dtype=torch.float), inv_freq).float()
- return torch.cat((torch.sin(sinusoid_inp), torch.cos(sinusoid_inp)), dim=1)
-
-
-# Copied from transformers.models.gptj.modeling_gptj.rotate_every_two
-def rotate_every_two(x: torch.Tensor) -> torch.Tensor:
- x1 = x[:, :, :, ::2]
- x2 = x[:, :, :, 1::2]
- x = torch.stack((-x2, x1), dim=-1)
- return x.flatten(-2) # in einsum notation: rearrange(x, '... d j -> ... (d j)')
-
-
-# Copied from transformers.models.gptj.modeling_gptj.apply_rotary_pos_emb
-def apply_rotary_pos_emb(tensor: torch.Tensor, sin: torch.Tensor, cos: torch.Tensor) -> torch.Tensor:
- sin = torch.repeat_interleave(sin[:, :, None, :], 2, 3)
- cos = torch.repeat_interleave(cos[:, :, None, :], 2, 3)
- return (tensor * cos) + (rotate_every_two(tensor) * sin)
-
-
-class MossAttention(nn.Module):
- def __init__(self, config):
- super().__init__()
-
- max_positions = config.max_position_embeddings
- self.register_buffer(
- "causal_mask",
- torch.tril(torch.ones((max_positions, max_positions), dtype=torch.bool)).view(
- 1, 1, max_positions, max_positions
- ),
- )
-
- self.attn_dropout = nn.Dropout(config.attn_pdrop)
- self.resid_dropout = nn.Dropout(config.resid_pdrop)
-
- self.embed_dim = config.hidden_size
- self.num_attention_heads = config.num_attention_heads
- self.head_dim = self.embed_dim // self.num_attention_heads
- if self.head_dim * self.num_attention_heads != self.embed_dim:
- raise ValueError(
- f"embed_dim must be divisible by num_attention_heads (got `embed_dim`: {self.embed_dim} and"
- f" `num_attention_heads`: {self.num_attention_heads})."
- )
- self.scale_attn = torch.sqrt(torch.tensor(self.head_dim, dtype=torch.float32)).to(torch.get_default_dtype())
- self.qkv_proj = nn.Linear(self.embed_dim, self.embed_dim * 3, bias=False)
-
- self.out_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=False)
- self.rotary_dim = config.rotary_dim
- pos_embd_dim = self.rotary_dim or self.embed_dim
- self.embed_positions = create_sinusoidal_positions(max_positions, pos_embd_dim)
-
- def _split_heads(self, x, n_head, dim_head, mp_num):
- reshaped = x.reshape(x.shape[:-1] + (n_head // mp_num, dim_head))
- reshaped = reshaped.reshape(x.shape[:-2] + (-1,) + reshaped.shape[-1:])
- return reshaped
-
- def _merge_heads(self, tensor, num_attention_heads, attn_head_size):
- """
- Merges attn_head_size dim and num_attn_heads dim into n_ctx
- """
- if len(tensor.shape) == 5:
- tensor = tensor.permute(0, 1, 3, 2, 4).contiguous()
- elif len(tensor.shape) == 4:
- tensor = tensor.permute(0, 2, 1, 3).contiguous()
- else:
- raise ValueError(f"Input tensor rank should be one of [4, 5], but is: {len(tensor.shape)}")
- new_shape = tensor.size()[:-2] + (num_attention_heads * attn_head_size,)
- return tensor.view(new_shape)
-
- def _attn(
- self,
- query,
- key,
- value,
- attention_mask=None,
- head_mask=None,
- ):
- # compute causal mask from causal mask buffer
- query_length, key_length = query.size(-2), key.size(-2)
- causal_mask = self.causal_mask[:, :, key_length - query_length : key_length, :key_length]
-
- # Keep the attention weights computation in fp32 to avoid overflow issues
- query = query.to(torch.float32)
- key = key.to(torch.float32)
-
- attn_weights = torch.matmul(query, key.transpose(-1, -2))
-
- attn_weights = attn_weights / self.scale_attn
- mask_value = torch.finfo(attn_weights.dtype).min
- # Need to be a tensor, otherwise we get error: `RuntimeError: expected scalar type float but found double`.
- # Need to be on the same device, otherwise `RuntimeError: ..., x and y to be on the same device`
- mask_value = torch.tensor(mask_value, dtype=attn_weights.dtype).to(attn_weights.device)
- attn_weights = torch.where(causal_mask, attn_weights, mask_value)
-
- if attention_mask is not None:
- # Apply the attention mask
- attn_weights = attn_weights + attention_mask
-
- attn_weights = nn.Softmax(dim=-1)(attn_weights)
- attn_weights = attn_weights.to(value.dtype)
- attn_weights = self.attn_dropout(attn_weights)
-
- # Mask heads if we want to
- if head_mask is not None:
- attn_weights = attn_weights * head_mask
-
- attn_output = torch.matmul(attn_weights, value)
-
- return attn_output, attn_weights
-
- def forward(
- self,
- hidden_states: Optional[torch.FloatTensor],
- layer_past: Optional[Tuple[torch.Tensor]] = None,
- attention_mask: Optional[torch.FloatTensor] = None,
- position_ids: Optional[torch.LongTensor] = None,
- head_mask: Optional[torch.FloatTensor] = None,
- use_cache: Optional[bool] = False,
- output_attentions: Optional[bool] = False,
- ) -> Union[
- Tuple[torch.Tensor, Tuple[torch.Tensor]],
- Optional[Tuple[torch.Tensor, Tuple[torch.Tensor], Tuple[torch.Tensor, ...]]],
- ]:
- qkv = self.qkv_proj(hidden_states)
- # TODO(enijkamp): factor out number of logical TPU-v4 cores or make forward pass agnostic
- mp_num = 4
- qkv_split = qkv.reshape(qkv.shape[:-1] + (mp_num, -1))
-
- local_dim = self.head_dim * self.num_attention_heads // mp_num
- query, value, key = torch.split(qkv_split, local_dim, dim=-1)
- query = self._split_heads(query, self.num_attention_heads, self.head_dim, mp_num=mp_num)
- key = self._split_heads(key, self.num_attention_heads, self.head_dim, mp_num=mp_num)
-
- value = self._split_heads(value, self.num_attention_heads, self.head_dim, mp_num=mp_num)
- value = value.permute(0, 2, 1, 3)
-
- embed_positions = self.embed_positions
- if embed_positions.device != position_ids.device:
- embed_positions = embed_positions.to(position_ids.device)
- self.embed_positions = embed_positions
-
- sincos = embed_positions[position_ids]
- sin, cos = torch.split(sincos, sincos.shape[-1] // 2, dim=-1)
-
- if self.rotary_dim is not None:
- k_rot = key[:, :, :, : self.rotary_dim]
- k_pass = key[:, :, :, self.rotary_dim :]
-
- q_rot = query[:, :, :, : self.rotary_dim]
- q_pass = query[:, :, :, self.rotary_dim :]
-
- k_rot = apply_rotary_pos_emb(k_rot, sin, cos)
- q_rot = apply_rotary_pos_emb(q_rot, sin, cos)
-
- key = torch.cat([k_rot, k_pass], dim=-1)
- query = torch.cat([q_rot, q_pass], dim=-1)
- else:
- key = apply_rotary_pos_emb(key, sin, cos)
- query = apply_rotary_pos_emb(query, sin, cos)
-
- key = key.permute(0, 2, 1, 3)
- query = query.permute(0, 2, 1, 3)
-
- if layer_past is not None:
- past_key = layer_past[0]
- past_value = layer_past[1]
- key = torch.cat((past_key, key), dim=-2)
- value = torch.cat((past_value, value), dim=-2)
-
- if use_cache is True:
- present = (key, value)
- else:
- present = None
-
- # compute self-attention: V x Softmax(QK^T)
- attn_output, attn_weights = self._attn(query, key, value, attention_mask, head_mask)
-
- attn_output = self._merge_heads(attn_output, self.num_attention_heads, self.head_dim)
- attn_output = self.out_proj(attn_output)
- attn_output = self.resid_dropout(attn_output)
-
- outputs = (attn_output, present)
- if output_attentions:
- outputs += (attn_weights,)
-
- return outputs # a, present, (attentions)
-
-
-# Copied from transformers.models.gptj.modeling_gptj.GPTJMLP with GPTJ->Moss
-class MossMLP(nn.Module):
- def __init__(self, intermediate_size, config): # in MLP: intermediate_size= 4 * embed_dim
- super().__init__()
- embed_dim = config.n_embd
-
- self.fc_in = nn.Linear(embed_dim, intermediate_size)
- self.fc_out = nn.Linear(intermediate_size, embed_dim)
-
- self.act = ACT2FN[config.activation_function]
- self.dropout = nn.Dropout(config.resid_pdrop)
-
- def forward(self, hidden_states: Optional[torch.FloatTensor]) -> torch.FloatTensor:
- hidden_states = self.fc_in(hidden_states)
- hidden_states = self.act(hidden_states)
- hidden_states = self.fc_out(hidden_states)
- hidden_states = self.dropout(hidden_states)
- return hidden_states
-
-
-# Copied from transformers.models.gptj.modeling_gptj.GPTJBlock with GPTJ->Moss
-class MossBlock(nn.Module):
- def __init__(self, config):
- super().__init__()
- inner_dim = config.n_inner if config.n_inner is not None else 4 * config.n_embd
- self.ln_1 = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
- self.attn = MossAttention(config)
- self.mlp = MossMLP(inner_dim, config)
-
- def forward(
- self,
- hidden_states: Optional[torch.FloatTensor],
- layer_past: Optional[Tuple[torch.Tensor]] = None,
- attention_mask: Optional[torch.FloatTensor] = None,
- position_ids: Optional[torch.LongTensor] = None,
- head_mask: Optional[torch.FloatTensor] = None,
- use_cache: Optional[bool] = False,
- output_attentions: Optional[bool] = False,
- ) -> Union[Tuple[torch.Tensor], Optional[Tuple[torch.Tensor, Tuple[torch.FloatTensor, ...]]]]:
- residual = hidden_states
- hidden_states = self.ln_1(hidden_states)
- attn_outputs = self.attn(
- hidden_states=hidden_states,
- layer_past=layer_past,
- attention_mask=attention_mask,
- position_ids=position_ids,
- head_mask=head_mask,
- use_cache=use_cache,
- output_attentions=output_attentions,
- )
- attn_output = attn_outputs[0] # output_attn: a, present, (attentions)
- outputs = attn_outputs[1:]
-
- feed_forward_hidden_states = self.mlp(hidden_states)
- hidden_states = attn_output + feed_forward_hidden_states + residual
-
- if use_cache:
- outputs = (hidden_states,) + outputs
- else:
- outputs = (hidden_states,) + outputs[1:]
-
- return outputs # hidden_states, present, (attentions)
-
-
-class MossPreTrainedModel(PreTrainedModel):
- """
- An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
- models.
- """
-
- config_class = MossConfig
- base_model_prefix = "transformer"
- supports_gradient_checkpointing = True
- _no_split_modules = ["MossBlock"]
-
- def __init__(self, *inputs, **kwargs):
- super().__init__(*inputs, **kwargs)
-
- def _init_weights(self, module):
- """Initialize the weights."""
- if isinstance(module, (nn.Linear,)):
- # Slightly different from Mesh Transformer JAX which uses truncated_normal for initialization
- # cf https://github.com/pytorch/pytorch/pull/5617
- module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
- if module.bias is not None:
- module.bias.data.zero_()
- elif isinstance(module, nn.Embedding):
- module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
- if module.padding_idx is not None:
- module.weight.data[module.padding_idx].zero_()
- elif isinstance(module, nn.LayerNorm):
- module.bias.data.zero_()
- module.weight.data.fill_(1.0)
-
- def _set_gradient_checkpointing(self, module, value=False):
- if isinstance(module, MossModel):
- module.gradient_checkpointing = value
-
-
-MOSS_START_DOCSTRING = r"""
- This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
- it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
- behavior.
-
- Parameters:
- config ([`MossConfig`]): Model configuration class with all the parameters of the model.
- Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
-"""
-
-MOSS_INPUTS_DOCSTRING = r"""
- Args:
- input_ids (`torch.LongTensor` of shape `({0})`):
- Indices of input sequence tokens in the vocabulary.
-
- Indices can be obtained using [`AutoProcenizer`]. See [`PreTrainedTokenizer.encode`] and
- [`PreTrainedTokenizer.__call__`] for details.
-
- [What are input IDs?](../glossary#input-ids)
- attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
- Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
-
- - 1 for tokens that are **not masked**,
- - 0 for tokens that are **masked**.
-
- [What are attention masks?](../glossary#attention-mask)
- token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
- 1]`:
-
- - 0 corresponds to a *sentence A* token,
- - 1 corresponds to a *sentence B* token.
-
- [What are token type IDs?](../glossary#token-type-ids)
- position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
- config.n_positions - 1]`.
-
- [What are position IDs?](../glossary#position-ids)
- head_mask (`torch.FloatTensor` of shape `(num_attention_heads,)` or `(n_layer, num_attention_heads)`, *optional*):
- Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:
-
- - 1 indicates the head is **not masked**,
- - 0 indicates the head is **masked**.
-
- inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_dim)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
- is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
- model's internal embedding lookup matrix.
- output_attentions (`bool`, *optional*):
- Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
- tensors for more detail.
- output_hidden_states (`bool`, *optional*):
- Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
- more detail.
- return_dict (`bool`, *optional*):
- Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
-"""
-
-
-@add_start_docstrings(
- "The bare Moss Model transformer outputting raw hidden-states without any specific head on top.",
- MOSS_START_DOCSTRING,
-)
-class MossModel(MossPreTrainedModel):
- def __init__(self, config):
- super().__init__(config)
-
- self.embed_dim = config.n_embd
- self.vocab_size = config.vocab_size
- self.wte = nn.Embedding(config.vocab_size, self.embed_dim)
- self.drop = nn.Dropout(config.embd_pdrop)
- self.h = nn.ModuleList([MossBlock(config) for _ in range(config.n_layer)])
- self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
- self.rotary_dim = min(config.rotary_dim, config.n_ctx // config.num_attention_heads)
-
- self.gradient_checkpointing = False
-
- # Initialize weights and apply final processing
- self.post_init()
-
- def get_input_embeddings(self):
- return self.wte
-
- def set_input_embeddings(self, new_embeddings):
- self.wte = new_embeddings
-
- @add_start_docstrings_to_model_forward(MOSS_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
- @add_code_sample_docstrings(
- checkpoint=_CHECKPOINT_FOR_DOC,
- output_type=BaseModelOutputWithPast,
- config_class=_CONFIG_FOR_DOC,
- )
- def forward(
- self,
- input_ids: Optional[torch.LongTensor] = None,
- past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
- attention_mask: Optional[torch.FloatTensor] = None,
- token_type_ids: Optional[torch.LongTensor] = None,
- position_ids: Optional[torch.LongTensor] = None,
- head_mask: Optional[torch.FloatTensor] = None,
- inputs_embeds: Optional[torch.FloatTensor] = None,
- use_cache: Optional[bool] = None,
- output_attentions: Optional[bool] = None,
- output_hidden_states: Optional[bool] = None,
- return_dict: Optional[bool] = None,
- ) -> Union[Tuple, BaseModelOutputWithPast]:
- output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
- output_hidden_states = (
- output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
- )
- use_cache = use_cache if use_cache is not None else self.config.use_cache
- return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-
- if input_ids is not None and inputs_embeds is not None:
- raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
- elif input_ids is not None:
- input_shape = input_ids.size()
- input_ids = input_ids.view(-1, input_shape[-1])
- batch_size = input_ids.shape[0]
- elif inputs_embeds is not None:
- input_shape = inputs_embeds.size()[:-1]
- batch_size = inputs_embeds.shape[0]
- else:
- raise ValueError("You have to specify either input_ids or inputs_embeds")
-
- device = input_ids.device if input_ids is not None else inputs_embeds.device
-
- if token_type_ids is not None:
- token_type_ids = token_type_ids.view(-1, input_shape[-1])
-
- if position_ids is not None:
- position_ids = position_ids.view(-1, input_shape[-1]).long()
-
- if past_key_values is None:
- past_length = 0
- past_key_values = tuple([None] * len(self.h))
- else:
- past_length = past_key_values[0][0].size(-2)
-
- if position_ids is None:
- position_ids = torch.arange(past_length, input_shape[-1] + past_length, dtype=torch.long, device=device)
- position_ids = position_ids.unsqueeze(0).view(-1, input_shape[-1])
-
- # Attention mask.
- if attention_mask is not None:
- if batch_size <= 0:
- raise ValueError("batch_size has to be defined and > 0")
- attention_mask = attention_mask.view(batch_size, -1)
- # We create a 3D attention mask from a 2D tensor mask.
- # Sizes are [batch_size, 1, 1, to_seq_length]
- # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length]
- # this attention mask is more simple than the triangular masking of causal attention
- # used in OpenAI GPT, we just need to prepare the broadcast dimension here.
- attention_mask = attention_mask[:, None, None, :]
-
- # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
- # masked positions, this operation will create a tensor which is 0.0 for
- # positions we want to attend and the dtype's smallest value for masked positions.
- # Since we are adding it to the raw scores before the softmax, this is
- # effectively the same as removing these entirely.
- attention_mask = attention_mask.to(dtype=self.dtype) # fp16 compatibility
- attention_mask = (1.0 - attention_mask) * torch.finfo(self.dtype).min
-
- # Prepare head mask if needed
- # 1.0 in head_mask indicate we keep the head
- # attention_probs has shape bsz x num_attention_heads x N x N
- # head_mask has shape n_layer x batch x num_attention_heads x N x N
- head_mask = self.get_head_mask(head_mask, self.config.n_layer)
-
- if inputs_embeds is None:
- inputs_embeds = self.wte(input_ids)
-
- hidden_states = inputs_embeds
-
- if token_type_ids is not None:
- token_type_embeds = self.wte(token_type_ids)
- hidden_states = hidden_states + token_type_embeds
-
- hidden_states = self.drop(hidden_states)
-
- output_shape = input_shape + (hidden_states.size(-1),)
-
- if self.gradient_checkpointing and self.training:
- if use_cache:
- logger.warning_once(
- "`use_cache=True` is incompatible with `config.gradient_checkpointing=True`. Setting "
- "`use_cache=False`..."
- )
- use_cache = False
-
- presents = () if use_cache else None
- all_self_attentions = () if output_attentions else None
- all_hidden_states = () if output_hidden_states else None
- for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)):
- if output_hidden_states:
- all_hidden_states = all_hidden_states + (hidden_states,)
-
- if self.gradient_checkpointing and self.training:
-
- def create_custom_forward(module):
- def custom_forward(*inputs):
- # None for past_key_value
- return module(*inputs, use_cache, output_attentions)
-
- return custom_forward
-
- outputs = torch.utils.checkpoint.checkpoint(
- create_custom_forward(block),
- hidden_states,
- None,
- attention_mask,
- position_ids,
- head_mask[i],
- )
- else:
- outputs = block(
- hidden_states=hidden_states,
- layer_past=layer_past,
- attention_mask=attention_mask,
- position_ids=position_ids,
- head_mask=head_mask[i],
- use_cache=use_cache,
- output_attentions=output_attentions,
- )
-
- hidden_states = outputs[0]
- if use_cache is True:
- presents = presents + (outputs[1],)
-
- if output_attentions:
- all_self_attentions = all_self_attentions + (outputs[2 if use_cache else 1],)
-
- hidden_states = self.ln_f(hidden_states)
-
- hidden_states = hidden_states.view(output_shape)
- # Add last hidden state
- if output_hidden_states:
- all_hidden_states = all_hidden_states + (hidden_states,)
-
- if not return_dict:
- return tuple(v for v in [hidden_states, presents, all_hidden_states, all_self_attentions] if v is not None)
-
- return BaseModelOutputWithPast(
- last_hidden_state=hidden_states,
- past_key_values=presents,
- hidden_states=all_hidden_states,
- attentions=all_self_attentions,
- )
-
-
-@add_start_docstrings(
- """
- The Moss Model transformer with a language modeling head on top.
- """,
- MOSS_START_DOCSTRING,
-)
-class MossForCausalLM(MossPreTrainedModel):
- _keys_to_ignore_on_load_missing = [r"h\.\d+\.attn\.causal_mask"]
-
- def __init__(self, config):
- super().__init__(config)
- if not hasattr(config, 'wbits'):
- config.wbits = 32
- config.groupsize = 128
-
- if config.wbits not in [4, 8, 32]:
- logger.warning(f'Specify `wbits` with 4, 8 or 32 to load the model. ')
- if config.wbits in [4, 8]:
- def noop(*args, **kwargs):
- pass
- torch.nn.init.kaiming_uniform_ = noop
- torch.nn.init.uniform_ = noop
- torch.nn.init.normal_ = noop
-
- torch.set_default_dtype(torch.half)
- transformers.modeling_utils._init_weights = False
- torch.set_default_dtype(torch.half)
- self.transformer = MossModel(config)
- self.lm_head = nn.Linear(config.n_embd, config.vocab_size)
- if config.wbits in [4, 8]:
- torch.set_default_dtype(torch.float)
- transformers.modeling_utils._init_weights = True
- self.quantize(config.wbits, config.groupsize)
- # Initialize weights and apply final processing
- self.post_init()
-
- def get_output_embeddings(self):
- return self.lm_head
-
- def set_output_embeddings(self, new_embeddings):
- self.lm_head = new_embeddings
-
- def prepare_inputs_for_generation(self, input_ids, past_key_values=None, **kwargs):
- token_type_ids = kwargs.get("token_type_ids", None)
- # only last token for inputs_ids if past is defined in kwargs
- if past_key_values:
- input_ids = input_ids[:, -1].unsqueeze(-1)
- if token_type_ids is not None:
- token_type_ids = token_type_ids[:, -1].unsqueeze(-1)
-
- attention_mask = kwargs.get("attention_mask", None)
- position_ids = kwargs.get("position_ids", None)
-
- if attention_mask is not None and position_ids is None:
- # create position_ids on the fly for batch generation
- position_ids = attention_mask.long().cumsum(-1) - 1
- position_ids.masked_fill_(attention_mask == 0, 1)
- if past_key_values:
- position_ids = position_ids[:, -1].unsqueeze(-1)
-
- return {
- "input_ids": input_ids,
- "past_key_values": past_key_values,
- "use_cache": kwargs.get("use_cache"),
- "position_ids": position_ids,
- "attention_mask": attention_mask,
- "token_type_ids": token_type_ids,
- }
-
- @add_start_docstrings_to_model_forward(MOSS_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
- @add_code_sample_docstrings(
- checkpoint=_CHECKPOINT_FOR_DOC,
- output_type=CausalLMOutputWithPast,
- config_class=_CONFIG_FOR_DOC,
- )
- def forward(
- self,
- input_ids: Optional[torch.LongTensor] = None,
- past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
- attention_mask: Optional[torch.FloatTensor] = None,
- token_type_ids: Optional[torch.LongTensor] = None,
- position_ids: Optional[torch.LongTensor] = None,
- head_mask: Optional[torch.FloatTensor] = None,
- inputs_embeds: Optional[torch.FloatTensor] = None,
- labels: Optional[torch.LongTensor] = None,
- use_cache: Optional[bool] = None,
- output_attentions: Optional[bool] = None,
- output_hidden_states: Optional[bool] = None,
- return_dict: Optional[bool] = None,
- ) -> Union[Tuple, CausalLMOutputWithPast]:
- r"""
- labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
- `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
- are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
- """
- return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-
- transformer_outputs = self.transformer(
- input_ids,
- past_key_values=past_key_values,
- attention_mask=attention_mask,
- token_type_ids=token_type_ids,
- position_ids=position_ids,
- head_mask=head_mask,
- inputs_embeds=inputs_embeds,
- use_cache=use_cache,
- output_attentions=output_attentions,
- output_hidden_states=output_hidden_states,
- return_dict=return_dict,
- )
- hidden_states = transformer_outputs[0]
-
- # make sure sampling in fp16 works correctly and
- # compute loss in fp32 to match with mesh-tf version
- # https://github.com/EleutherAI/gpt-neo/blob/89ce74164da2fb16179106f54e2269b5da8db333/models/gpt2/gpt2.py#L179
- lm_logits = self.lm_head(hidden_states).to(torch.float32)
-
- loss = None
- if labels is not None:
- # Shift so that tokens < n predict n
- shift_logits = lm_logits[..., :-1, :].contiguous()
- shift_labels = labels[..., 1:].contiguous()
- # Flatten the tokens
- loss_fct = CrossEntropyLoss()
- loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
-
- loss = loss.to(hidden_states.dtype)
-
- if not return_dict:
- output = (lm_logits,) + transformer_outputs[1:]
- return ((loss,) + output) if loss is not None else output
-
- return CausalLMOutputWithPast(
- loss=loss,
- logits=lm_logits,
- past_key_values=transformer_outputs.past_key_values,
- hidden_states=transformer_outputs.hidden_states,
- attentions=transformer_outputs.attentions,
- )
-
- @staticmethod
- def _reorder_cache(
- past_key_values: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor
- ) -> Tuple[Tuple[torch.Tensor]]:
- """
- This function is used to re-order the `past_key_values` cache if [`~PretrainedModel.beam_search`] or
- [`~PretrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct
- beam_idx at every generation step.
- """
- return tuple(
- tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past)
- for layer_past in past_key_values
- )
-
- def quantize(self, wbits, groupsize):
- from .quantization import quantize_with_gptq
- return quantize_with_gptq(self, wbits, groupsize)
-
-""" PyTorch Moss model."""
-
-from typing import Optional, Tuple, Union
-
-import torch
-import torch.utils.checkpoint
-from torch import nn
-from torch.nn import CrossEntropyLoss
-import transformers
-from transformers.activations import ACT2FN
-from transformers.modeling_utils import PreTrainedModel
-from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
-from transformers.utils import (
- add_code_sample_docstrings,
- add_start_docstrings,
- add_start_docstrings_to_model_forward,
- logging
-)
-
-from .configuration_moss import MossConfig
-
-logger = logging.get_logger(__name__)
-
-_CHECKPOINT_FOR_DOC = "fnlp/moss-moon-003-base"
-_CONFIG_FOR_DOC = "MossConfig"
-
-
-MOSS_PRETRAINED_MODEL_ARCHIVE_LIST = [
- "fnlp/moss-moon-003-base",
- "fnlp/moss-moon-003-sft",
- "fnlp/moss-moon-003-sft-plugin",
- "fnlp/moss-moon-003-sft-int4",
- "fnlp/moss-moon-003-sft-plugin-int4",
- "fnlp/moss-moon-003-sft-int8",
- "fnlp/moss-moon-003-sft-plugin-int8",
-]
-
-
-# Copied from transformers.models.gptj.modeling_gptj.create_sinusoidal_positions
-def create_sinusoidal_positions(num_pos: int, dim: int) -> torch.Tensor:
- inv_freq = 1.0 / (10000 ** (torch.arange(0, dim, 2) / dim))
- sinusoid_inp = torch.einsum("i , j -> i j", torch.arange(num_pos, dtype=torch.float), inv_freq).float()
- return torch.cat((torch.sin(sinusoid_inp), torch.cos(sinusoid_inp)), dim=1)
-
-
-# Copied from transformers.models.gptj.modeling_gptj.rotate_every_two
-def rotate_every_two(x: torch.Tensor) -> torch.Tensor:
- x1 = x[:, :, :, ::2]
- x2 = x[:, :, :, 1::2]
- x = torch.stack((-x2, x1), dim=-1)
- return x.flatten(-2) # in einsum notation: rearrange(x, '... d j -> ... (d j)')
-
-
-# Copied from transformers.models.gptj.modeling_gptj.apply_rotary_pos_emb
-def apply_rotary_pos_emb(tensor: torch.Tensor, sin: torch.Tensor, cos: torch.Tensor) -> torch.Tensor:
- sin = torch.repeat_interleave(sin[:, :, None, :], 2, 3)
- cos = torch.repeat_interleave(cos[:, :, None, :], 2, 3)
- return (tensor * cos) + (rotate_every_two(tensor) * sin)
-
-
-class MossAttention(nn.Module):
- def __init__(self, config):
- super().__init__()
-
- max_positions = config.max_position_embeddings
- self.register_buffer(
- "causal_mask",
- torch.tril(torch.ones((max_positions, max_positions), dtype=torch.bool)).view(
- 1, 1, max_positions, max_positions
- ),
- )
-
- self.attn_dropout = nn.Dropout(config.attn_pdrop)
- self.resid_dropout = nn.Dropout(config.resid_pdrop)
-
- self.embed_dim = config.hidden_size
- self.num_attention_heads = config.num_attention_heads
- self.head_dim = self.embed_dim // self.num_attention_heads
- if self.head_dim * self.num_attention_heads != self.embed_dim:
- raise ValueError(
- f"embed_dim must be divisible by num_attention_heads (got `embed_dim`: {self.embed_dim} and"
- f" `num_attention_heads`: {self.num_attention_heads})."
- )
- self.scale_attn = torch.sqrt(torch.tensor(self.head_dim, dtype=torch.float32)).to(torch.get_default_dtype())
- self.qkv_proj = nn.Linear(self.embed_dim, self.embed_dim * 3, bias=False)
-
- self.out_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=False)
- self.rotary_dim = config.rotary_dim
- pos_embd_dim = self.rotary_dim or self.embed_dim
- self.embed_positions = create_sinusoidal_positions(max_positions, pos_embd_dim)
-
- def _split_heads(self, x, n_head, dim_head, mp_num):
- reshaped = x.reshape(x.shape[:-1] + (n_head // mp_num, dim_head))
- reshaped = reshaped.reshape(x.shape[:-2] + (-1,) + reshaped.shape[-1:])
- return reshaped
-
- def _merge_heads(self, tensor, num_attention_heads, attn_head_size):
- """
- Merges attn_head_size dim and num_attn_heads dim into n_ctx
- """
- if len(tensor.shape) == 5:
- tensor = tensor.permute(0, 1, 3, 2, 4).contiguous()
- elif len(tensor.shape) == 4:
- tensor = tensor.permute(0, 2, 1, 3).contiguous()
- else:
- raise ValueError(f"Input tensor rank should be one of [4, 5], but is: {len(tensor.shape)}")
- new_shape = tensor.size()[:-2] + (num_attention_heads * attn_head_size,)
- return tensor.view(new_shape)
-
- def _attn(
- self,
- query,
- key,
- value,
- attention_mask=None,
- head_mask=None,
- ):
- # compute causal mask from causal mask buffer
- query_length, key_length = query.size(-2), key.size(-2)
- causal_mask = self.causal_mask[:, :, key_length - query_length : key_length, :key_length]
-
- # Keep the attention weights computation in fp32 to avoid overflow issues
- query = query.to(torch.float32)
- key = key.to(torch.float32)
-
- attn_weights = torch.matmul(query, key.transpose(-1, -2))
-
- attn_weights = attn_weights / self.scale_attn
- mask_value = torch.finfo(attn_weights.dtype).min
- # Need to be a tensor, otherwise we get error: `RuntimeError: expected scalar type float but found double`.
- # Need to be on the same device, otherwise `RuntimeError: ..., x and y to be on the same device`
- mask_value = torch.tensor(mask_value, dtype=attn_weights.dtype).to(attn_weights.device)
- attn_weights = torch.where(causal_mask, attn_weights, mask_value)
-
- if attention_mask is not None:
- # Apply the attention mask
- attn_weights = attn_weights + attention_mask
-
- attn_weights = nn.Softmax(dim=-1)(attn_weights)
- attn_weights = attn_weights.to(value.dtype)
- attn_weights = self.attn_dropout(attn_weights)
-
- # Mask heads if we want to
- if head_mask is not None:
- attn_weights = attn_weights * head_mask
-
- attn_output = torch.matmul(attn_weights, value)
-
- return attn_output, attn_weights
-
- def forward(
- self,
- hidden_states: Optional[torch.FloatTensor],
- layer_past: Optional[Tuple[torch.Tensor]] = None,
- attention_mask: Optional[torch.FloatTensor] = None,
- position_ids: Optional[torch.LongTensor] = None,
- head_mask: Optional[torch.FloatTensor] = None,
- use_cache: Optional[bool] = False,
- output_attentions: Optional[bool] = False,
- ) -> Union[
- Tuple[torch.Tensor, Tuple[torch.Tensor]],
- Optional[Tuple[torch.Tensor, Tuple[torch.Tensor], Tuple[torch.Tensor, ...]]],
- ]:
- qkv = self.qkv_proj(hidden_states)
- # TODO(enijkamp): factor out number of logical TPU-v4 cores or make forward pass agnostic
- mp_num = 4
- qkv_split = qkv.reshape(qkv.shape[:-1] + (mp_num, -1))
-
- local_dim = self.head_dim * self.num_attention_heads // mp_num
- query, value, key = torch.split(qkv_split, local_dim, dim=-1)
- query = self._split_heads(query, self.num_attention_heads, self.head_dim, mp_num=mp_num)
- key = self._split_heads(key, self.num_attention_heads, self.head_dim, mp_num=mp_num)
-
- value = self._split_heads(value, self.num_attention_heads, self.head_dim, mp_num=mp_num)
- value = value.permute(0, 2, 1, 3)
-
- embed_positions = self.embed_positions
- if embed_positions.device != position_ids.device:
- embed_positions = embed_positions.to(position_ids.device)
- self.embed_positions = embed_positions
-
- sincos = embed_positions[position_ids]
- sin, cos = torch.split(sincos, sincos.shape[-1] // 2, dim=-1)
-
- if self.rotary_dim is not None:
- k_rot = key[:, :, :, : self.rotary_dim]
- k_pass = key[:, :, :, self.rotary_dim :]
-
- q_rot = query[:, :, :, : self.rotary_dim]
- q_pass = query[:, :, :, self.rotary_dim :]
-
- k_rot = apply_rotary_pos_emb(k_rot, sin, cos)
- q_rot = apply_rotary_pos_emb(q_rot, sin, cos)
-
- key = torch.cat([k_rot, k_pass], dim=-1)
- query = torch.cat([q_rot, q_pass], dim=-1)
- else:
- key = apply_rotary_pos_emb(key, sin, cos)
- query = apply_rotary_pos_emb(query, sin, cos)
-
- key = key.permute(0, 2, 1, 3)
- query = query.permute(0, 2, 1, 3)
-
- if layer_past is not None:
- past_key = layer_past[0]
- past_value = layer_past[1]
- key = torch.cat((past_key, key), dim=-2)
- value = torch.cat((past_value, value), dim=-2)
-
- if use_cache is True:
- present = (key, value)
- else:
- present = None
-
- # compute self-attention: V x Softmax(QK^T)
- attn_output, attn_weights = self._attn(query, key, value, attention_mask, head_mask)
-
- attn_output = self._merge_heads(attn_output, self.num_attention_heads, self.head_dim)
- attn_output = self.out_proj(attn_output)
- attn_output = self.resid_dropout(attn_output)
-
- outputs = (attn_output, present)
- if output_attentions:
- outputs += (attn_weights,)
-
- return outputs # a, present, (attentions)
-
-
-# Copied from transformers.models.gptj.modeling_gptj.GPTJMLP with GPTJ->Moss
-class MossMLP(nn.Module):
- def __init__(self, intermediate_size, config): # in MLP: intermediate_size= 4 * embed_dim
- super().__init__()
- embed_dim = config.n_embd
-
- self.fc_in = nn.Linear(embed_dim, intermediate_size)
- self.fc_out = nn.Linear(intermediate_size, embed_dim)
-
- self.act = ACT2FN[config.activation_function]
- self.dropout = nn.Dropout(config.resid_pdrop)
-
- def forward(self, hidden_states: Optional[torch.FloatTensor]) -> torch.FloatTensor:
- hidden_states = self.fc_in(hidden_states)
- hidden_states = self.act(hidden_states)
- hidden_states = self.fc_out(hidden_states)
- hidden_states = self.dropout(hidden_states)
- return hidden_states
-
-
-# Copied from transformers.models.gptj.modeling_gptj.GPTJBlock with GPTJ->Moss
-class MossBlock(nn.Module):
- def __init__(self, config):
- super().__init__()
- inner_dim = config.n_inner if config.n_inner is not None else 4 * config.n_embd
- self.ln_1 = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
- self.attn = MossAttention(config)
- self.mlp = MossMLP(inner_dim, config)
-
- def forward(
- self,
- hidden_states: Optional[torch.FloatTensor],
- layer_past: Optional[Tuple[torch.Tensor]] = None,
- attention_mask: Optional[torch.FloatTensor] = None,
- position_ids: Optional[torch.LongTensor] = None,
- head_mask: Optional[torch.FloatTensor] = None,
- use_cache: Optional[bool] = False,
- output_attentions: Optional[bool] = False,
- ) -> Union[Tuple[torch.Tensor], Optional[Tuple[torch.Tensor, Tuple[torch.FloatTensor, ...]]]]:
- residual = hidden_states
- hidden_states = self.ln_1(hidden_states)
- attn_outputs = self.attn(
- hidden_states=hidden_states,
- layer_past=layer_past,
- attention_mask=attention_mask,
- position_ids=position_ids,
- head_mask=head_mask,
- use_cache=use_cache,
- output_attentions=output_attentions,
- )
- attn_output = attn_outputs[0] # output_attn: a, present, (attentions)
- outputs = attn_outputs[1:]
-
- feed_forward_hidden_states = self.mlp(hidden_states)
- hidden_states = attn_output + feed_forward_hidden_states + residual
-
- if use_cache:
- outputs = (hidden_states,) + outputs
- else:
- outputs = (hidden_states,) + outputs[1:]
-
- return outputs # hidden_states, present, (attentions)
-
-
-class MossPreTrainedModel(PreTrainedModel):
- """
- An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
- models.
- """
-
- config_class = MossConfig
- base_model_prefix = "transformer"
- supports_gradient_checkpointing = True
- _no_split_modules = ["MossBlock"]
-
- def __init__(self, *inputs, **kwargs):
- super().__init__(*inputs, **kwargs)
-
- def _init_weights(self, module):
- """Initialize the weights."""
- if isinstance(module, (nn.Linear,)):
- # Slightly different from Mesh Transformer JAX which uses truncated_normal for initialization
- # cf https://github.com/pytorch/pytorch/pull/5617
- module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
- if module.bias is not None:
- module.bias.data.zero_()
- elif isinstance(module, nn.Embedding):
- module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
- if module.padding_idx is not None:
- module.weight.data[module.padding_idx].zero_()
- elif isinstance(module, nn.LayerNorm):
- module.bias.data.zero_()
- module.weight.data.fill_(1.0)
-
- def _set_gradient_checkpointing(self, module, value=False):
- if isinstance(module, MossModel):
- module.gradient_checkpointing = value
-
-
-MOSS_START_DOCSTRING = r"""
- This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
- it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
- behavior.
-
- Parameters:
- config ([`MossConfig`]): Model configuration class with all the parameters of the model.
- Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
-"""
-
-MOSS_INPUTS_DOCSTRING = r"""
- Args:
- input_ids (`torch.LongTensor` of shape `({0})`):
- Indices of input sequence tokens in the vocabulary.
-
- Indices can be obtained using [`AutoProcenizer`]. See [`PreTrainedTokenizer.encode`] and
- [`PreTrainedTokenizer.__call__`] for details.
-
- [What are input IDs?](../glossary#input-ids)
- attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
- Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
-
- - 1 for tokens that are **not masked**,
- - 0 for tokens that are **masked**.
-
- [What are attention masks?](../glossary#attention-mask)
- token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
- 1]`:
-
- - 0 corresponds to a *sentence A* token,
- - 1 corresponds to a *sentence B* token.
-
- [What are token type IDs?](../glossary#token-type-ids)
- position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
- config.n_positions - 1]`.
-
- [What are position IDs?](../glossary#position-ids)
- head_mask (`torch.FloatTensor` of shape `(num_attention_heads,)` or `(n_layer, num_attention_heads)`, *optional*):
- Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:
-
- - 1 indicates the head is **not masked**,
- - 0 indicates the head is **masked**.
-
- inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_dim)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
- is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
- model's internal embedding lookup matrix.
- output_attentions (`bool`, *optional*):
- Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
- tensors for more detail.
- output_hidden_states (`bool`, *optional*):
- Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
- more detail.
- return_dict (`bool`, *optional*):
- Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
-"""
-
-
-@add_start_docstrings(
- "The bare Moss Model transformer outputting raw hidden-states without any specific head on top.",
- MOSS_START_DOCSTRING,
-)
-class MossModel(MossPreTrainedModel):
- def __init__(self, config):
- super().__init__(config)
-
- self.embed_dim = config.n_embd
- self.vocab_size = config.vocab_size
- self.wte = nn.Embedding(config.vocab_size, self.embed_dim)
- self.drop = nn.Dropout(config.embd_pdrop)
- self.h = nn.ModuleList([MossBlock(config) for _ in range(config.n_layer)])
- self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
- self.rotary_dim = min(config.rotary_dim, config.n_ctx // config.num_attention_heads)
-
- self.gradient_checkpointing = False
-
- # Initialize weights and apply final processing
- self.post_init()
-
- def get_input_embeddings(self):
- return self.wte
-
- def set_input_embeddings(self, new_embeddings):
- self.wte = new_embeddings
-
- @add_start_docstrings_to_model_forward(MOSS_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
- @add_code_sample_docstrings(
- checkpoint=_CHECKPOINT_FOR_DOC,
- output_type=BaseModelOutputWithPast,
- config_class=_CONFIG_FOR_DOC,
- )
- def forward(
- self,
- input_ids: Optional[torch.LongTensor] = None,
- past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
- attention_mask: Optional[torch.FloatTensor] = None,
- token_type_ids: Optional[torch.LongTensor] = None,
- position_ids: Optional[torch.LongTensor] = None,
- head_mask: Optional[torch.FloatTensor] = None,
- inputs_embeds: Optional[torch.FloatTensor] = None,
- use_cache: Optional[bool] = None,
- output_attentions: Optional[bool] = None,
- output_hidden_states: Optional[bool] = None,
- return_dict: Optional[bool] = None,
- ) -> Union[Tuple, BaseModelOutputWithPast]:
- output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
- output_hidden_states = (
- output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
- )
- use_cache = use_cache if use_cache is not None else self.config.use_cache
- return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-
- if input_ids is not None and inputs_embeds is not None:
- raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
- elif input_ids is not None:
- input_shape = input_ids.size()
- input_ids = input_ids.view(-1, input_shape[-1])
- batch_size = input_ids.shape[0]
- elif inputs_embeds is not None:
- input_shape = inputs_embeds.size()[:-1]
- batch_size = inputs_embeds.shape[0]
- else:
- raise ValueError("You have to specify either input_ids or inputs_embeds")
-
- device = input_ids.device if input_ids is not None else inputs_embeds.device
-
- if token_type_ids is not None:
- token_type_ids = token_type_ids.view(-1, input_shape[-1])
-
- if position_ids is not None:
- position_ids = position_ids.view(-1, input_shape[-1]).long()
-
- if past_key_values is None:
- past_length = 0
- past_key_values = tuple([None] * len(self.h))
- else:
- past_length = past_key_values[0][0].size(-2)
-
- if position_ids is None:
- position_ids = torch.arange(past_length, input_shape[-1] + past_length, dtype=torch.long, device=device)
- position_ids = position_ids.unsqueeze(0).view(-1, input_shape[-1])
-
- # Attention mask.
- if attention_mask is not None:
- if batch_size <= 0:
- raise ValueError("batch_size has to be defined and > 0")
- attention_mask = attention_mask.view(batch_size, -1)
- # We create a 3D attention mask from a 2D tensor mask.
- # Sizes are [batch_size, 1, 1, to_seq_length]
- # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length]
- # this attention mask is more simple than the triangular masking of causal attention
- # used in OpenAI GPT, we just need to prepare the broadcast dimension here.
- attention_mask = attention_mask[:, None, None, :]
-
- # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
- # masked positions, this operation will create a tensor which is 0.0 for
- # positions we want to attend and the dtype's smallest value for masked positions.
- # Since we are adding it to the raw scores before the softmax, this is
- # effectively the same as removing these entirely.
- attention_mask = attention_mask.to(dtype=self.dtype) # fp16 compatibility
- attention_mask = (1.0 - attention_mask) * torch.finfo(self.dtype).min
-
- # Prepare head mask if needed
- # 1.0 in head_mask indicate we keep the head
- # attention_probs has shape bsz x num_attention_heads x N x N
- # head_mask has shape n_layer x batch x num_attention_heads x N x N
- head_mask = self.get_head_mask(head_mask, self.config.n_layer)
-
- if inputs_embeds is None:
- inputs_embeds = self.wte(input_ids)
-
- hidden_states = inputs_embeds
-
- if token_type_ids is not None:
- token_type_embeds = self.wte(token_type_ids)
- hidden_states = hidden_states + token_type_embeds
-
- hidden_states = self.drop(hidden_states)
-
- output_shape = input_shape + (hidden_states.size(-1),)
-
- if self.gradient_checkpointing and self.training:
- if use_cache:
- logger.warning_once(
- "`use_cache=True` is incompatible with `config.gradient_checkpointing=True`. Setting "
- "`use_cache=False`..."
- )
- use_cache = False
-
- presents = () if use_cache else None
- all_self_attentions = () if output_attentions else None
- all_hidden_states = () if output_hidden_states else None
- for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)):
- if output_hidden_states:
- all_hidden_states = all_hidden_states + (hidden_states,)
-
- if self.gradient_checkpointing and self.training:
-
- def create_custom_forward(module):
- def custom_forward(*inputs):
- # None for past_key_value
- return module(*inputs, use_cache, output_attentions)
-
- return custom_forward
-
- outputs = torch.utils.checkpoint.checkpoint(
- create_custom_forward(block),
- hidden_states,
- None,
- attention_mask,
- position_ids,
- head_mask[i],
- )
- else:
- outputs = block(
- hidden_states=hidden_states,
- layer_past=layer_past,
- attention_mask=attention_mask,
- position_ids=position_ids,
- head_mask=head_mask[i],
- use_cache=use_cache,
- output_attentions=output_attentions,
- )
-
- hidden_states = outputs[0]
- if use_cache is True:
- presents = presents + (outputs[1],)
-
- if output_attentions:
- all_self_attentions = all_self_attentions + (outputs[2 if use_cache else 1],)
-
- hidden_states = self.ln_f(hidden_states)
-
- hidden_states = hidden_states.view(output_shape)
- # Add last hidden state
- if output_hidden_states:
- all_hidden_states = all_hidden_states + (hidden_states,)
-
- if not return_dict:
- return tuple(v for v in [hidden_states, presents, all_hidden_states, all_self_attentions] if v is not None)
-
- return BaseModelOutputWithPast(
- last_hidden_state=hidden_states,
- past_key_values=presents,
- hidden_states=all_hidden_states,
- attentions=all_self_attentions,
- )
-
-
-@add_start_docstrings(
- """
- The Moss Model transformer with a language modeling head on top.
- """,
- MOSS_START_DOCSTRING,
-)
-class MossForCausalLM(MossPreTrainedModel):
- _keys_to_ignore_on_load_missing = [r"h\.\d+\.attn\.causal_mask"]
-
- def __init__(self, config):
- super().__init__(config)
- if not hasattr(config, 'wbits'):
- config.wbits = 32
- config.groupsize = 128
-
- if config.wbits not in [4, 8, 32]:
- logger.warning(f'Specify `wbits` with 4, 8 or 32 to load the model. ')
- if config.wbits in [4, 8]:
- def noop(*args, **kwargs):
- pass
- torch.nn.init.kaiming_uniform_ = noop
- torch.nn.init.uniform_ = noop
- torch.nn.init.normal_ = noop
-
- torch.set_default_dtype(torch.half)
- transformers.modeling_utils._init_weights = False
- torch.set_default_dtype(torch.half)
- self.transformer = MossModel(config)
- self.lm_head = nn.Linear(config.n_embd, config.vocab_size)
- if config.wbits in [4, 8]:
- torch.set_default_dtype(torch.float)
- transformers.modeling_utils._init_weights = True
- self.quantize(config.wbits, config.groupsize)
- # Initialize weights and apply final processing
- self.post_init()
-
- def get_output_embeddings(self):
- return self.lm_head
-
- def set_output_embeddings(self, new_embeddings):
- self.lm_head = new_embeddings
-
- def prepare_inputs_for_generation(self, input_ids, past_key_values=None, **kwargs):
- token_type_ids = kwargs.get("token_type_ids", None)
- # only last token for inputs_ids if past is defined in kwargs
- if past_key_values:
- input_ids = input_ids[:, -1].unsqueeze(-1)
- if token_type_ids is not None:
- token_type_ids = token_type_ids[:, -1].unsqueeze(-1)
-
- attention_mask = kwargs.get("attention_mask", None)
- position_ids = kwargs.get("position_ids", None)
-
- if attention_mask is not None and position_ids is None:
- # create position_ids on the fly for batch generation
- position_ids = attention_mask.long().cumsum(-1) - 1
- position_ids.masked_fill_(attention_mask == 0, 1)
- if past_key_values:
- position_ids = position_ids[:, -1].unsqueeze(-1)
-
- return {
- "input_ids": input_ids,
- "past_key_values": past_key_values,
- "use_cache": kwargs.get("use_cache"),
- "position_ids": position_ids,
- "attention_mask": attention_mask,
- "token_type_ids": token_type_ids,
- }
-
- @add_start_docstrings_to_model_forward(MOSS_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
- @add_code_sample_docstrings(
- checkpoint=_CHECKPOINT_FOR_DOC,
- output_type=CausalLMOutputWithPast,
- config_class=_CONFIG_FOR_DOC,
- )
- def forward(
- self,
- input_ids: Optional[torch.LongTensor] = None,
- past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
- attention_mask: Optional[torch.FloatTensor] = None,
- token_type_ids: Optional[torch.LongTensor] = None,
- position_ids: Optional[torch.LongTensor] = None,
- head_mask: Optional[torch.FloatTensor] = None,
- inputs_embeds: Optional[torch.FloatTensor] = None,
- labels: Optional[torch.LongTensor] = None,
- use_cache: Optional[bool] = None,
- output_attentions: Optional[bool] = None,
- output_hidden_states: Optional[bool] = None,
- return_dict: Optional[bool] = None,
- ) -> Union[Tuple, CausalLMOutputWithPast]:
- r"""
- labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
- `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
- are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
- """
- return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-
- transformer_outputs = self.transformer(
- input_ids,
- past_key_values=past_key_values,
- attention_mask=attention_mask,
- token_type_ids=token_type_ids,
- position_ids=position_ids,
- head_mask=head_mask,
- inputs_embeds=inputs_embeds,
- use_cache=use_cache,
- output_attentions=output_attentions,
- output_hidden_states=output_hidden_states,
- return_dict=return_dict,
- )
- hidden_states = transformer_outputs[0]
-
- # make sure sampling in fp16 works correctly and
- # compute loss in fp32 to match with mesh-tf version
- # https://github.com/EleutherAI/gpt-neo/blob/89ce74164da2fb16179106f54e2269b5da8db333/models/gpt2/gpt2.py#L179
- lm_logits = self.lm_head(hidden_states).to(torch.float32)
-
- loss = None
- if labels is not None:
- # Shift so that tokens < n predict n
- shift_logits = lm_logits[..., :-1, :].contiguous()
- shift_labels = labels[..., 1:].contiguous()
- # Flatten the tokens
- loss_fct = CrossEntropyLoss()
- loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
-
- loss = loss.to(hidden_states.dtype)
-
- if not return_dict:
- output = (lm_logits,) + transformer_outputs[1:]
- return ((loss,) + output) if loss is not None else output
-
- return CausalLMOutputWithPast(
- loss=loss,
- logits=lm_logits,
- past_key_values=transformer_outputs.past_key_values,
- hidden_states=transformer_outputs.hidden_states,
- attentions=transformer_outputs.attentions,
- )
-
- @staticmethod
- def _reorder_cache(
- past_key_values: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor
- ) -> Tuple[Tuple[torch.Tensor]]:
- """
- This function is used to re-order the `past_key_values` cache if [`~PretrainedModel.beam_search`] or
- [`~PretrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct
- beam_idx at every generation step.
- """
- return tuple(
- tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past)
- for layer_past in past_key_values
- )
-
- def quantize(self, wbits, groupsize):
- from .quantization import quantize_with_gptq
- return quantize_with_gptq(self, wbits, groupsize)
-
-""" PyTorch Moss model."""
-
-from typing import Optional, Tuple, Union
-
-import torch
-import torch.utils.checkpoint
-from torch import nn
-from torch.nn import CrossEntropyLoss
-import transformers
-from transformers.activations import ACT2FN
-from transformers.modeling_utils import PreTrainedModel
-from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
-from transformers.utils import (
- add_code_sample_docstrings,
- add_start_docstrings,
- add_start_docstrings_to_model_forward,
- logging
-)
-
-from .configuration_moss import MossConfig
-
-logger = logging.get_logger(__name__)
-
-_CHECKPOINT_FOR_DOC = "fnlp/moss-moon-003-base"
-_CONFIG_FOR_DOC = "MossConfig"
-
-
-MOSS_PRETRAINED_MODEL_ARCHIVE_LIST = [
- "fnlp/moss-moon-003-base",
- "fnlp/moss-moon-003-sft",
- "fnlp/moss-moon-003-sft-plugin",
- "fnlp/moss-moon-003-sft-int4",
- "fnlp/moss-moon-003-sft-plugin-int4",
- "fnlp/moss-moon-003-sft-int8",
- "fnlp/moss-moon-003-sft-plugin-int8",
-]
-
-
-# Copied from transformers.models.gptj.modeling_gptj.create_sinusoidal_positions
-def create_sinusoidal_positions(num_pos: int, dim: int) -> torch.Tensor:
- inv_freq = 1.0 / (10000 ** (torch.arange(0, dim, 2) / dim))
- sinusoid_inp = torch.einsum("i , j -> i j", torch.arange(num_pos, dtype=torch.float), inv_freq).float()
- return torch.cat((torch.sin(sinusoid_inp), torch.cos(sinusoid_inp)), dim=1)
-
-
-# Copied from transformers.models.gptj.modeling_gptj.rotate_every_two
-def rotate_every_two(x: torch.Tensor) -> torch.Tensor:
- x1 = x[:, :, :, ::2]
- x2 = x[:, :, :, 1::2]
- x = torch.stack((-x2, x1), dim=-1)
- return x.flatten(-2) # in einsum notation: rearrange(x, '... d j -> ... (d j)')
-
-
-# Copied from transformers.models.gptj.modeling_gptj.apply_rotary_pos_emb
-def apply_rotary_pos_emb(tensor: torch.Tensor, sin: torch.Tensor, cos: torch.Tensor) -> torch.Tensor:
- sin = torch.repeat_interleave(sin[:, :, None, :], 2, 3)
- cos = torch.repeat_interleave(cos[:, :, None, :], 2, 3)
- return (tensor * cos) + (rotate_every_two(tensor) * sin)
-
-
-class MossAttention(nn.Module):
- def __init__(self, config):
- super().__init__()
-
- max_positions = config.max_position_embeddings
- self.register_buffer(
- "causal_mask",
- torch.tril(torch.ones((max_positions, max_positions), dtype=torch.bool)).view(
- 1, 1, max_positions, max_positions
- ),
- )
-
- self.attn_dropout = nn.Dropout(config.attn_pdrop)
- self.resid_dropout = nn.Dropout(config.resid_pdrop)
-
- self.embed_dim = config.hidden_size
- self.num_attention_heads = config.num_attention_heads
- self.head_dim = self.embed_dim // self.num_attention_heads
- if self.head_dim * self.num_attention_heads != self.embed_dim:
- raise ValueError(
- f"embed_dim must be divisible by num_attention_heads (got `embed_dim`: {self.embed_dim} and"
- f" `num_attention_heads`: {self.num_attention_heads})."
- )
- self.scale_attn = torch.sqrt(torch.tensor(self.head_dim, dtype=torch.float32)).to(torch.get_default_dtype())
- self.qkv_proj = nn.Linear(self.embed_dim, self.embed_dim * 3, bias=False)
-
- self.out_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=False)
- self.rotary_dim = config.rotary_dim
- pos_embd_dim = self.rotary_dim or self.embed_dim
- self.embed_positions = create_sinusoidal_positions(max_positions, pos_embd_dim)
-
- def _split_heads(self, x, n_head, dim_head, mp_num):
- reshaped = x.reshape(x.shape[:-1] + (n_head // mp_num, dim_head))
- reshaped = reshaped.reshape(x.shape[:-2] + (-1,) + reshaped.shape[-1:])
- return reshaped
-
- def _merge_heads(self, tensor, num_attention_heads, attn_head_size):
- """
- Merges attn_head_size dim and num_attn_heads dim into n_ctx
- """
- if len(tensor.shape) == 5:
- tensor = tensor.permute(0, 1, 3, 2, 4).contiguous()
- elif len(tensor.shape) == 4:
- tensor = tensor.permute(0, 2, 1, 3).contiguous()
- else:
- raise ValueError(f"Input tensor rank should be one of [4, 5], but is: {len(tensor.shape)}")
- new_shape = tensor.size()[:-2] + (num_attention_heads * attn_head_size,)
- return tensor.view(new_shape)
-
- def _attn(
- self,
- query,
- key,
- value,
- attention_mask=None,
- head_mask=None,
- ):
- # compute causal mask from causal mask buffer
- query_length, key_length = query.size(-2), key.size(-2)
- causal_mask = self.causal_mask[:, :, key_length - query_length : key_length, :key_length]
-
- # Keep the attention weights computation in fp32 to avoid overflow issues
- query = query.to(torch.float32)
- key = key.to(torch.float32)
-
- attn_weights = torch.matmul(query, key.transpose(-1, -2))
-
- attn_weights = attn_weights / self.scale_attn
- mask_value = torch.finfo(attn_weights.dtype).min
- # Need to be a tensor, otherwise we get error: `RuntimeError: expected scalar type float but found double`.
- # Need to be on the same device, otherwise `RuntimeError: ..., x and y to be on the same device`
- mask_value = torch.tensor(mask_value, dtype=attn_weights.dtype).to(attn_weights.device)
- attn_weights = torch.where(causal_mask, attn_weights, mask_value)
-
- if attention_mask is not None:
- # Apply the attention mask
- attn_weights = attn_weights + attention_mask
-
- attn_weights = nn.Softmax(dim=-1)(attn_weights)
- attn_weights = attn_weights.to(value.dtype)
- attn_weights = self.attn_dropout(attn_weights)
-
- # Mask heads if we want to
- if head_mask is not None:
- attn_weights = attn_weights * head_mask
-
- attn_output = torch.matmul(attn_weights, value)
-
- return attn_output, attn_weights
-
- def forward(
- self,
- hidden_states: Optional[torch.FloatTensor],
- layer_past: Optional[Tuple[torch.Tensor]] = None,
- attention_mask: Optional[torch.FloatTensor] = None,
- position_ids: Optional[torch.LongTensor] = None,
- head_mask: Optional[torch.FloatTensor] = None,
- use_cache: Optional[bool] = False,
- output_attentions: Optional[bool] = False,
- ) -> Union[
- Tuple[torch.Tensor, Tuple[torch.Tensor]],
- Optional[Tuple[torch.Tensor, Tuple[torch.Tensor], Tuple[torch.Tensor, ...]]],
- ]:
- qkv = self.qkv_proj(hidden_states)
- # TODO(enijkamp): factor out number of logical TPU-v4 cores or make forward pass agnostic
- mp_num = 4
- qkv_split = qkv.reshape(qkv.shape[:-1] + (mp_num, -1))
-
- local_dim = self.head_dim * self.num_attention_heads // mp_num
- query, value, key = torch.split(qkv_split, local_dim, dim=-1)
- query = self._split_heads(query, self.num_attention_heads, self.head_dim, mp_num=mp_num)
- key = self._split_heads(key, self.num_attention_heads, self.head_dim, mp_num=mp_num)
-
- value = self._split_heads(value, self.num_attention_heads, self.head_dim, mp_num=mp_num)
- value = value.permute(0, 2, 1, 3)
-
- embed_positions = self.embed_positions
- if embed_positions.device != position_ids.device:
- embed_positions = embed_positions.to(position_ids.device)
- self.embed_positions = embed_positions
-
- sincos = embed_positions[position_ids]
- sin, cos = torch.split(sincos, sincos.shape[-1] // 2, dim=-1)
-
- if self.rotary_dim is not None:
- k_rot = key[:, :, :, : self.rotary_dim]
- k_pass = key[:, :, :, self.rotary_dim :]
-
- q_rot = query[:, :, :, : self.rotary_dim]
- q_pass = query[:, :, :, self.rotary_dim :]
-
- k_rot = apply_rotary_pos_emb(k_rot, sin, cos)
- q_rot = apply_rotary_pos_emb(q_rot, sin, cos)
-
- key = torch.cat([k_rot, k_pass], dim=-1)
- query = torch.cat([q_rot, q_pass], dim=-1)
- else:
- key = apply_rotary_pos_emb(key, sin, cos)
- query = apply_rotary_pos_emb(query, sin, cos)
-
- key = key.permute(0, 2, 1, 3)
- query = query.permute(0, 2, 1, 3)
-
- if layer_past is not None:
- past_key = layer_past[0]
- past_value = layer_past[1]
- key = torch.cat((past_key, key), dim=-2)
- value = torch.cat((past_value, value), dim=-2)
-
- if use_cache is True:
- present = (key, value)
- else:
- present = None
-
- # compute self-attention: V x Softmax(QK^T)
- attn_output, attn_weights = self._attn(query, key, value, attention_mask, head_mask)
-
- attn_output = self._merge_heads(attn_output, self.num_attention_heads, self.head_dim)
- attn_output = self.out_proj(attn_output)
- attn_output = self.resid_dropout(attn_output)
-
- outputs = (attn_output, present)
- if output_attentions:
- outputs += (attn_weights,)
-
- return outputs # a, present, (attentions)
-
-
-# Copied from transformers.models.gptj.modeling_gptj.GPTJMLP with GPTJ->Moss
-class MossMLP(nn.Module):
- def __init__(self, intermediate_size, config): # in MLP: intermediate_size= 4 * embed_dim
- super().__init__()
- embed_dim = config.n_embd
-
- self.fc_in = nn.Linear(embed_dim, intermediate_size)
- self.fc_out = nn.Linear(intermediate_size, embed_dim)
-
- self.act = ACT2FN[config.activation_function]
- self.dropout = nn.Dropout(config.resid_pdrop)
-
- def forward(self, hidden_states: Optional[torch.FloatTensor]) -> torch.FloatTensor:
- hidden_states = self.fc_in(hidden_states)
- hidden_states = self.act(hidden_states)
- hidden_states = self.fc_out(hidden_states)
- hidden_states = self.dropout(hidden_states)
- return hidden_states
-
-
-# Copied from transformers.models.gptj.modeling_gptj.GPTJBlock with GPTJ->Moss
-class MossBlock(nn.Module):
- def __init__(self, config):
- super().__init__()
- inner_dim = config.n_inner if config.n_inner is not None else 4 * config.n_embd
- self.ln_1 = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
- self.attn = MossAttention(config)
- self.mlp = MossMLP(inner_dim, config)
-
- def forward(
- self,
- hidden_states: Optional[torch.FloatTensor],
- layer_past: Optional[Tuple[torch.Tensor]] = None,
- attention_mask: Optional[torch.FloatTensor] = None,
- position_ids: Optional[torch.LongTensor] = None,
- head_mask: Optional[torch.FloatTensor] = None,
- use_cache: Optional[bool] = False,
- output_attentions: Optional[bool] = False,
- ) -> Union[Tuple[torch.Tensor], Optional[Tuple[torch.Tensor, Tuple[torch.FloatTensor, ...]]]]:
- residual = hidden_states
- hidden_states = self.ln_1(hidden_states)
- attn_outputs = self.attn(
- hidden_states=hidden_states,
- layer_past=layer_past,
- attention_mask=attention_mask,
- position_ids=position_ids,
- head_mask=head_mask,
- use_cache=use_cache,
- output_attentions=output_attentions,
- )
- attn_output = attn_outputs[0] # output_attn: a, present, (attentions)
- outputs = attn_outputs[1:]
-
- feed_forward_hidden_states = self.mlp(hidden_states)
- hidden_states = attn_output + feed_forward_hidden_states + residual
-
- if use_cache:
- outputs = (hidden_states,) + outputs
- else:
- outputs = (hidden_states,) + outputs[1:]
-
- return outputs # hidden_states, present, (attentions)
-
-
-class MossPreTrainedModel(PreTrainedModel):
- """
- An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
- models.
- """
-
- config_class = MossConfig
- base_model_prefix = "transformer"
- supports_gradient_checkpointing = True
- _no_split_modules = ["MossBlock"]
-
- def __init__(self, *inputs, **kwargs):
- super().__init__(*inputs, **kwargs)
-
- def _init_weights(self, module):
- """Initialize the weights."""
- if isinstance(module, (nn.Linear,)):
- # Slightly different from Mesh Transformer JAX which uses truncated_normal for initialization
- # cf https://github.com/pytorch/pytorch/pull/5617
- module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
- if module.bias is not None:
- module.bias.data.zero_()
- elif isinstance(module, nn.Embedding):
- module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
- if module.padding_idx is not None:
- module.weight.data[module.padding_idx].zero_()
- elif isinstance(module, nn.LayerNorm):
- module.bias.data.zero_()
- module.weight.data.fill_(1.0)
-
- def _set_gradient_checkpointing(self, module, value=False):
- if isinstance(module, MossModel):
- module.gradient_checkpointing = value
-
-
-MOSS_START_DOCSTRING = r"""
- This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
- it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
- behavior.
-
- Parameters:
- config ([`MossConfig`]): Model configuration class with all the parameters of the model.
- Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
-"""
-
-MOSS_INPUTS_DOCSTRING = r"""
- Args:
- input_ids (`torch.LongTensor` of shape `({0})`):
- Indices of input sequence tokens in the vocabulary.
-
- Indices can be obtained using [`AutoProcenizer`]. See [`PreTrainedTokenizer.encode`] and
- [`PreTrainedTokenizer.__call__`] for details.
-
- [What are input IDs?](../glossary#input-ids)
- attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
- Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
-
- - 1 for tokens that are **not masked**,
- - 0 for tokens that are **masked**.
-
- [What are attention masks?](../glossary#attention-mask)
- token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
- 1]`:
-
- - 0 corresponds to a *sentence A* token,
- - 1 corresponds to a *sentence B* token.
-
- [What are token type IDs?](../glossary#token-type-ids)
- position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
- config.n_positions - 1]`.
-
- [What are position IDs?](../glossary#position-ids)
- head_mask (`torch.FloatTensor` of shape `(num_attention_heads,)` or `(n_layer, num_attention_heads)`, *optional*):
- Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:
-
- - 1 indicates the head is **not masked**,
- - 0 indicates the head is **masked**.
-
- inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_dim)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
- is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
- model's internal embedding lookup matrix.
- output_attentions (`bool`, *optional*):
- Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
- tensors for more detail.
- output_hidden_states (`bool`, *optional*):
- Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
- more detail.
- return_dict (`bool`, *optional*):
- Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
-"""
-
-
-@add_start_docstrings(
- "The bare Moss Model transformer outputting raw hidden-states without any specific head on top.",
- MOSS_START_DOCSTRING,
-)
-class MossModel(MossPreTrainedModel):
- def __init__(self, config):
- super().__init__(config)
-
- self.embed_dim = config.n_embd
- self.vocab_size = config.vocab_size
- self.wte = nn.Embedding(config.vocab_size, self.embed_dim)
- self.drop = nn.Dropout(config.embd_pdrop)
- self.h = nn.ModuleList([MossBlock(config) for _ in range(config.n_layer)])
- self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
- self.rotary_dim = min(config.rotary_dim, config.n_ctx // config.num_attention_heads)
-
- self.gradient_checkpointing = False
-
- # Initialize weights and apply final processing
- self.post_init()
-
- def get_input_embeddings(self):
- return self.wte
-
- def set_input_embeddings(self, new_embeddings):
- self.wte = new_embeddings
-
- @add_start_docstrings_to_model_forward(MOSS_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
- @add_code_sample_docstrings(
- checkpoint=_CHECKPOINT_FOR_DOC,
- output_type=BaseModelOutputWithPast,
- config_class=_CONFIG_FOR_DOC,
- )
- def forward(
- self,
- input_ids: Optional[torch.LongTensor] = None,
- past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
- attention_mask: Optional[torch.FloatTensor] = None,
- token_type_ids: Optional[torch.LongTensor] = None,
- position_ids: Optional[torch.LongTensor] = None,
- head_mask: Optional[torch.FloatTensor] = None,
- inputs_embeds: Optional[torch.FloatTensor] = None,
- use_cache: Optional[bool] = None,
- output_attentions: Optional[bool] = None,
- output_hidden_states: Optional[bool] = None,
- return_dict: Optional[bool] = None,
- ) -> Union[Tuple, BaseModelOutputWithPast]:
- output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
- output_hidden_states = (
- output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
- )
- use_cache = use_cache if use_cache is not None else self.config.use_cache
- return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-
- if input_ids is not None and inputs_embeds is not None:
- raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
- elif input_ids is not None:
- input_shape = input_ids.size()
- input_ids = input_ids.view(-1, input_shape[-1])
- batch_size = input_ids.shape[0]
- elif inputs_embeds is not None:
- input_shape = inputs_embeds.size()[:-1]
- batch_size = inputs_embeds.shape[0]
- else:
- raise ValueError("You have to specify either input_ids or inputs_embeds")
-
- device = input_ids.device if input_ids is not None else inputs_embeds.device
-
- if token_type_ids is not None:
- token_type_ids = token_type_ids.view(-1, input_shape[-1])
-
- if position_ids is not None:
- position_ids = position_ids.view(-1, input_shape[-1]).long()
-
- if past_key_values is None:
- past_length = 0
- past_key_values = tuple([None] * len(self.h))
- else:
- past_length = past_key_values[0][0].size(-2)
-
- if position_ids is None:
- position_ids = torch.arange(past_length, input_shape[-1] + past_length, dtype=torch.long, device=device)
- position_ids = position_ids.unsqueeze(0).view(-1, input_shape[-1])
-
- # Attention mask.
- if attention_mask is not None:
- if batch_size <= 0:
- raise ValueError("batch_size has to be defined and > 0")
- attention_mask = attention_mask.view(batch_size, -1)
- # We create a 3D attention mask from a 2D tensor mask.
- # Sizes are [batch_size, 1, 1, to_seq_length]
- # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length]
- # this attention mask is more simple than the triangular masking of causal attention
- # used in OpenAI GPT, we just need to prepare the broadcast dimension here.
- attention_mask = attention_mask[:, None, None, :]
-
- # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
- # masked positions, this operation will create a tensor which is 0.0 for
- # positions we want to attend and the dtype's smallest value for masked positions.
- # Since we are adding it to the raw scores before the softmax, this is
- # effectively the same as removing these entirely.
- attention_mask = attention_mask.to(dtype=self.dtype) # fp16 compatibility
- attention_mask = (1.0 - attention_mask) * torch.finfo(self.dtype).min
-
- # Prepare head mask if needed
- # 1.0 in head_mask indicate we keep the head
- # attention_probs has shape bsz x num_attention_heads x N x N
- # head_mask has shape n_layer x batch x num_attention_heads x N x N
- head_mask = self.get_head_mask(head_mask, self.config.n_layer)
-
- if inputs_embeds is None:
- inputs_embeds = self.wte(input_ids)
-
- hidden_states = inputs_embeds
-
- if token_type_ids is not None:
- token_type_embeds = self.wte(token_type_ids)
- hidden_states = hidden_states + token_type_embeds
-
- hidden_states = self.drop(hidden_states)
-
- output_shape = input_shape + (hidden_states.size(-1),)
-
- if self.gradient_checkpointing and self.training:
- if use_cache:
- logger.warning_once(
- "`use_cache=True` is incompatible with `config.gradient_checkpointing=True`. Setting "
- "`use_cache=False`..."
- )
- use_cache = False
-
- presents = () if use_cache else None
- all_self_attentions = () if output_attentions else None
- all_hidden_states = () if output_hidden_states else None
- for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)):
- if output_hidden_states:
- all_hidden_states = all_hidden_states + (hidden_states,)
-
- if self.gradient_checkpointing and self.training:
-
- def create_custom_forward(module):
- def custom_forward(*inputs):
- # None for past_key_value
- return module(*inputs, use_cache, output_attentions)
-
- return custom_forward
-
- outputs = torch.utils.checkpoint.checkpoint(
- create_custom_forward(block),
- hidden_states,
- None,
- attention_mask,
- position_ids,
- head_mask[i],
- )
- else:
- outputs = block(
- hidden_states=hidden_states,
- layer_past=layer_past,
- attention_mask=attention_mask,
- position_ids=position_ids,
- head_mask=head_mask[i],
- use_cache=use_cache,
- output_attentions=output_attentions,
- )
-
- hidden_states = outputs[0]
- if use_cache is True:
- presents = presents + (outputs[1],)
-
- if output_attentions:
- all_self_attentions = all_self_attentions + (outputs[2 if use_cache else 1],)
-
- hidden_states = self.ln_f(hidden_states)
-
- hidden_states = hidden_states.view(output_shape)
- # Add last hidden state
- if output_hidden_states:
- all_hidden_states = all_hidden_states + (hidden_states,)
-
- if not return_dict:
- return tuple(v for v in [hidden_states, presents, all_hidden_states, all_self_attentions] if v is not None)
-
- return BaseModelOutputWithPast(
- last_hidden_state=hidden_states,
- past_key_values=presents,
- hidden_states=all_hidden_states,
- attentions=all_self_attentions,
- )
-
-
-@add_start_docstrings(
- """
- The Moss Model transformer with a language modeling head on top.
- """,
- MOSS_START_DOCSTRING,
-)
-class MossForCausalLM(MossPreTrainedModel):
- _keys_to_ignore_on_load_missing = [r"h\.\d+\.attn\.causal_mask"]
-
- def __init__(self, config):
- super().__init__(config)
- if not hasattr(config, 'wbits'):
- config.wbits = 32
- config.groupsize = 128
-
- if config.wbits not in [4, 8, 32]:
- logger.warning(f'Specify `wbits` with 4, 8 or 32 to load the model. ')
- if config.wbits in [4, 8]:
- def noop(*args, **kwargs):
- pass
- torch.nn.init.kaiming_uniform_ = noop
- torch.nn.init.uniform_ = noop
- torch.nn.init.normal_ = noop
-
- torch.set_default_dtype(torch.half)
- transformers.modeling_utils._init_weights = False
- torch.set_default_dtype(torch.half)
- self.transformer = MossModel(config)
- self.lm_head = nn.Linear(config.n_embd, config.vocab_size)
- if config.wbits in [4, 8]:
- torch.set_default_dtype(torch.float)
- transformers.modeling_utils._init_weights = True
- self.quantize(config.wbits, config.groupsize)
- # Initialize weights and apply final processing
- self.post_init()
-
- def get_output_embeddings(self):
- return self.lm_head
-
- def set_output_embeddings(self, new_embeddings):
- self.lm_head = new_embeddings
-
- def prepare_inputs_for_generation(self, input_ids, past_key_values=None, **kwargs):
- token_type_ids = kwargs.get("token_type_ids", None)
- # only last token for inputs_ids if past is defined in kwargs
- if past_key_values:
- input_ids = input_ids[:, -1].unsqueeze(-1)
- if token_type_ids is not None:
- token_type_ids = token_type_ids[:, -1].unsqueeze(-1)
-
- attention_mask = kwargs.get("attention_mask", None)
- position_ids = kwargs.get("position_ids", None)
-
- if attention_mask is not None and position_ids is None:
- # create position_ids on the fly for batch generation
- position_ids = attention_mask.long().cumsum(-1) - 1
- position_ids.masked_fill_(attention_mask == 0, 1)
- if past_key_values:
- position_ids = position_ids[:, -1].unsqueeze(-1)
-
- return {
- "input_ids": input_ids,
- "past_key_values": past_key_values,
- "use_cache": kwargs.get("use_cache"),
- "position_ids": position_ids,
- "attention_mask": attention_mask,
- "token_type_ids": token_type_ids,
- }
-
- @add_start_docstrings_to_model_forward(MOSS_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
- @add_code_sample_docstrings(
- checkpoint=_CHECKPOINT_FOR_DOC,
- output_type=CausalLMOutputWithPast,
- config_class=_CONFIG_FOR_DOC,
- )
- def forward(
- self,
- input_ids: Optional[torch.LongTensor] = None,
- past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
- attention_mask: Optional[torch.FloatTensor] = None,
- token_type_ids: Optional[torch.LongTensor] = None,
- position_ids: Optional[torch.LongTensor] = None,
- head_mask: Optional[torch.FloatTensor] = None,
- inputs_embeds: Optional[torch.FloatTensor] = None,
- labels: Optional[torch.LongTensor] = None,
- use_cache: Optional[bool] = None,
- output_attentions: Optional[bool] = None,
- output_hidden_states: Optional[bool] = None,
- return_dict: Optional[bool] = None,
- ) -> Union[Tuple, CausalLMOutputWithPast]:
- r"""
- labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
- `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
- are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
- """
- return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-
- transformer_outputs = self.transformer(
- input_ids,
- past_key_values=past_key_values,
- attention_mask=attention_mask,
- token_type_ids=token_type_ids,
- position_ids=position_ids,
- head_mask=head_mask,
- inputs_embeds=inputs_embeds,
- use_cache=use_cache,
- output_attentions=output_attentions,
- output_hidden_states=output_hidden_states,
- return_dict=return_dict,
- )
- hidden_states = transformer_outputs[0]
-
- # make sure sampling in fp16 works correctly and
- # compute loss in fp32 to match with mesh-tf version
- # https://github.com/EleutherAI/gpt-neo/blob/89ce74164da2fb16179106f54e2269b5da8db333/models/gpt2/gpt2.py#L179
- lm_logits = self.lm_head(hidden_states).to(torch.float32)
-
- loss = None
- if labels is not None:
- # Shift so that tokens < n predict n
- shift_logits = lm_logits[..., :-1, :].contiguous()
- shift_labels = labels[..., 1:].contiguous()
- # Flatten the tokens
- loss_fct = CrossEntropyLoss()
- loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
-
- loss = loss.to(hidden_states.dtype)
-
- if not return_dict:
- output = (lm_logits,) + transformer_outputs[1:]
- return ((loss,) + output) if loss is not None else output
-
- return CausalLMOutputWithPast(
- loss=loss,
- logits=lm_logits,
- past_key_values=transformer_outputs.past_key_values,
- hidden_states=transformer_outputs.hidden_states,
- attentions=transformer_outputs.attentions,
- )
-
- @staticmethod
- def _reorder_cache(
- past_key_values: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor
- ) -> Tuple[Tuple[torch.Tensor]]:
- """
- This function is used to re-order the `past_key_values` cache if [`~PretrainedModel.beam_search`] or
- [`~PretrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct
- beam_idx at every generation step.
- """
- return tuple(
- tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past)
- for layer_past in past_key_values
- )
-
- def quantize(self, wbits, groupsize):
- from .quantization import quantize_with_gptq
- return quantize_with_gptq(self, wbits, groupsize)
-
-""" PyTorch Moss model."""
-
-from typing import Optional, Tuple, Union
-
-import torch
-import torch.utils.checkpoint
-from torch import nn
-from torch.nn import CrossEntropyLoss
-import transformers
-from transformers.activations import ACT2FN
-from transformers.modeling_utils import PreTrainedModel
-from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
-from transformers.utils import (
- add_code_sample_docstrings,
- add_start_docstrings,
- add_start_docstrings_to_model_forward,
- logging
-)
-
-from .configuration_moss import MossConfig
-
-logger = logging.get_logger(__name__)
-
-_CHECKPOINT_FOR_DOC = "fnlp/moss-moon-003-base"
-_CONFIG_FOR_DOC = "MossConfig"
-
-
-MOSS_PRETRAINED_MODEL_ARCHIVE_LIST = [
- "fnlp/moss-moon-003-base",
- "fnlp/moss-moon-003-sft",
- "fnlp/moss-moon-003-sft-plugin",
- "fnlp/moss-moon-003-sft-int4",
- "fnlp/moss-moon-003-sft-plugin-int4",
- "fnlp/moss-moon-003-sft-int8",
- "fnlp/moss-moon-003-sft-plugin-int8",
-]
-
-
-# Copied from transformers.models.gptj.modeling_gptj.create_sinusoidal_positions
-def create_sinusoidal_positions(num_pos: int, dim: int) -> torch.Tensor:
- inv_freq = 1.0 / (10000 ** (torch.arange(0, dim, 2) / dim))
- sinusoid_inp = torch.einsum("i , j -> i j", torch.arange(num_pos, dtype=torch.float), inv_freq).float()
- return torch.cat((torch.sin(sinusoid_inp), torch.cos(sinusoid_inp)), dim=1)
-
-
-# Copied from transformers.models.gptj.modeling_gptj.rotate_every_two
-def rotate_every_two(x: torch.Tensor) -> torch.Tensor:
- x1 = x[:, :, :, ::2]
- x2 = x[:, :, :, 1::2]
- x = torch.stack((-x2, x1), dim=-1)
- return x.flatten(-2) # in einsum notation: rearrange(x, '... d j -> ... (d j)')
-
-
-# Copied from transformers.models.gptj.modeling_gptj.apply_rotary_pos_emb
-def apply_rotary_pos_emb(tensor: torch.Tensor, sin: torch.Tensor, cos: torch.Tensor) -> torch.Tensor:
- sin = torch.repeat_interleave(sin[:, :, None, :], 2, 3)
- cos = torch.repeat_interleave(cos[:, :, None, :], 2, 3)
- return (tensor * cos) + (rotate_every_two(tensor) * sin)
-
-
-class MossAttention(nn.Module):
- def __init__(self, config):
- super().__init__()
-
- max_positions = config.max_position_embeddings
- self.register_buffer(
- "causal_mask",
- torch.tril(torch.ones((max_positions, max_positions), dtype=torch.bool)).view(
- 1, 1, max_positions, max_positions
- ),
- )
-
- self.attn_dropout = nn.Dropout(config.attn_pdrop)
- self.resid_dropout = nn.Dropout(config.resid_pdrop)
-
- self.embed_dim = config.hidden_size
- self.num_attention_heads = config.num_attention_heads
- self.head_dim = self.embed_dim // self.num_attention_heads
- if self.head_dim * self.num_attention_heads != self.embed_dim:
- raise ValueError(
- f"embed_dim must be divisible by num_attention_heads (got `embed_dim`: {self.embed_dim} and"
- f" `num_attention_heads`: {self.num_attention_heads})."
- )
- self.scale_attn = torch.sqrt(torch.tensor(self.head_dim, dtype=torch.float32)).to(torch.get_default_dtype())
- self.qkv_proj = nn.Linear(self.embed_dim, self.embed_dim * 3, bias=False)
-
- self.out_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=False)
- self.rotary_dim = config.rotary_dim
- pos_embd_dim = self.rotary_dim or self.embed_dim
- self.embed_positions = create_sinusoidal_positions(max_positions, pos_embd_dim)
-
- def _split_heads(self, x, n_head, dim_head, mp_num):
- reshaped = x.reshape(x.shape[:-1] + (n_head // mp_num, dim_head))
- reshaped = reshaped.reshape(x.shape[:-2] + (-1,) + reshaped.shape[-1:])
- return reshaped
-
- def _merge_heads(self, tensor, num_attention_heads, attn_head_size):
- """
- Merges attn_head_size dim and num_attn_heads dim into n_ctx
- """
- if len(tensor.shape) == 5:
- tensor = tensor.permute(0, 1, 3, 2, 4).contiguous()
- elif len(tensor.shape) == 4:
- tensor = tensor.permute(0, 2, 1, 3).contiguous()
- else:
- raise ValueError(f"Input tensor rank should be one of [4, 5], but is: {len(tensor.shape)}")
- new_shape = tensor.size()[:-2] + (num_attention_heads * attn_head_size,)
- return tensor.view(new_shape)
-
- def _attn(
- self,
- query,
- key,
- value,
- attention_mask=None,
- head_mask=None,
- ):
- # compute causal mask from causal mask buffer
- query_length, key_length = query.size(-2), key.size(-2)
- causal_mask = self.causal_mask[:, :, key_length - query_length : key_length, :key_length]
-
- # Keep the attention weights computation in fp32 to avoid overflow issues
- query = query.to(torch.float32)
- key = key.to(torch.float32)
-
- attn_weights = torch.matmul(query, key.transpose(-1, -2))
-
- attn_weights = attn_weights / self.scale_attn
- mask_value = torch.finfo(attn_weights.dtype).min
- # Need to be a tensor, otherwise we get error: `RuntimeError: expected scalar type float but found double`.
- # Need to be on the same device, otherwise `RuntimeError: ..., x and y to be on the same device`
- mask_value = torch.tensor(mask_value, dtype=attn_weights.dtype).to(attn_weights.device)
- attn_weights = torch.where(causal_mask, attn_weights, mask_value)
-
- if attention_mask is not None:
- # Apply the attention mask
- attn_weights = attn_weights + attention_mask
-
- attn_weights = nn.Softmax(dim=-1)(attn_weights)
- attn_weights = attn_weights.to(value.dtype)
- attn_weights = self.attn_dropout(attn_weights)
-
- # Mask heads if we want to
- if head_mask is not None:
- attn_weights = attn_weights * head_mask
-
- attn_output = torch.matmul(attn_weights, value)
-
- return attn_output, attn_weights
-
- def forward(
- self,
- hidden_states: Optional[torch.FloatTensor],
- layer_past: Optional[Tuple[torch.Tensor]] = None,
- attention_mask: Optional[torch.FloatTensor] = None,
- position_ids: Optional[torch.LongTensor] = None,
- head_mask: Optional[torch.FloatTensor] = None,
- use_cache: Optional[bool] = False,
- output_attentions: Optional[bool] = False,
- ) -> Union[
- Tuple[torch.Tensor, Tuple[torch.Tensor]],
- Optional[Tuple[torch.Tensor, Tuple[torch.Tensor], Tuple[torch.Tensor, ...]]],
- ]:
- qkv = self.qkv_proj(hidden_states)
- # TODO(enijkamp): factor out number of logical TPU-v4 cores or make forward pass agnostic
- mp_num = 4
- qkv_split = qkv.reshape(qkv.shape[:-1] + (mp_num, -1))
-
- local_dim = self.head_dim * self.num_attention_heads // mp_num
- query, value, key = torch.split(qkv_split, local_dim, dim=-1)
- query = self._split_heads(query, self.num_attention_heads, self.head_dim, mp_num=mp_num)
- key = self._split_heads(key, self.num_attention_heads, self.head_dim, mp_num=mp_num)
-
- value = self._split_heads(value, self.num_attention_heads, self.head_dim, mp_num=mp_num)
- value = value.permute(0, 2, 1, 3)
-
- embed_positions = self.embed_positions
- if embed_positions.device != position_ids.device:
- embed_positions = embed_positions.to(position_ids.device)
- self.embed_positions = embed_positions
-
- sincos = embed_positions[position_ids]
- sin, cos = torch.split(sincos, sincos.shape[-1] // 2, dim=-1)
-
- if self.rotary_dim is not None:
- k_rot = key[:, :, :, : self.rotary_dim]
- k_pass = key[:, :, :, self.rotary_dim :]
-
- q_rot = query[:, :, :, : self.rotary_dim]
- q_pass = query[:, :, :, self.rotary_dim :]
-
- k_rot = apply_rotary_pos_emb(k_rot, sin, cos)
- q_rot = apply_rotary_pos_emb(q_rot, sin, cos)
-
- key = torch.cat([k_rot, k_pass], dim=-1)
- query = torch.cat([q_rot, q_pass], dim=-1)
- else:
- key = apply_rotary_pos_emb(key, sin, cos)
- query = apply_rotary_pos_emb(query, sin, cos)
-
- key = key.permute(0, 2, 1, 3)
- query = query.permute(0, 2, 1, 3)
-
- if layer_past is not None:
- past_key = layer_past[0]
- past_value = layer_past[1]
- key = torch.cat((past_key, key), dim=-2)
- value = torch.cat((past_value, value), dim=-2)
-
- if use_cache is True:
- present = (key, value)
- else:
- present = None
-
- # compute self-attention: V x Softmax(QK^T)
- attn_output, attn_weights = self._attn(query, key, value, attention_mask, head_mask)
-
- attn_output = self._merge_heads(attn_output, self.num_attention_heads, self.head_dim)
- attn_output = self.out_proj(attn_output)
- attn_output = self.resid_dropout(attn_output)
-
- outputs = (attn_output, present)
- if output_attentions:
- outputs += (attn_weights,)
-
- return outputs # a, present, (attentions)
-
-
-# Copied from transformers.models.gptj.modeling_gptj.GPTJMLP with GPTJ->Moss
-class MossMLP(nn.Module):
- def __init__(self, intermediate_size, config): # in MLP: intermediate_size= 4 * embed_dim
- super().__init__()
- embed_dim = config.n_embd
-
- self.fc_in = nn.Linear(embed_dim, intermediate_size)
- self.fc_out = nn.Linear(intermediate_size, embed_dim)
-
- self.act = ACT2FN[config.activation_function]
- self.dropout = nn.Dropout(config.resid_pdrop)
-
- def forward(self, hidden_states: Optional[torch.FloatTensor]) -> torch.FloatTensor:
- hidden_states = self.fc_in(hidden_states)
- hidden_states = self.act(hidden_states)
- hidden_states = self.fc_out(hidden_states)
- hidden_states = self.dropout(hidden_states)
- return hidden_states
-
-
-# Copied from transformers.models.gptj.modeling_gptj.GPTJBlock with GPTJ->Moss
-class MossBlock(nn.Module):
- def __init__(self, config):
- super().__init__()
- inner_dim = config.n_inner if config.n_inner is not None else 4 * config.n_embd
- self.ln_1 = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
- self.attn = MossAttention(config)
- self.mlp = MossMLP(inner_dim, config)
-
- def forward(
- self,
- hidden_states: Optional[torch.FloatTensor],
- layer_past: Optional[Tuple[torch.Tensor]] = None,
- attention_mask: Optional[torch.FloatTensor] = None,
- position_ids: Optional[torch.LongTensor] = None,
- head_mask: Optional[torch.FloatTensor] = None,
- use_cache: Optional[bool] = False,
- output_attentions: Optional[bool] = False,
- ) -> Union[Tuple[torch.Tensor], Optional[Tuple[torch.Tensor, Tuple[torch.FloatTensor, ...]]]]:
- residual = hidden_states
- hidden_states = self.ln_1(hidden_states)
- attn_outputs = self.attn(
- hidden_states=hidden_states,
- layer_past=layer_past,
- attention_mask=attention_mask,
- position_ids=position_ids,
- head_mask=head_mask,
- use_cache=use_cache,
- output_attentions=output_attentions,
- )
- attn_output = attn_outputs[0] # output_attn: a, present, (attentions)
- outputs = attn_outputs[1:]
-
- feed_forward_hidden_states = self.mlp(hidden_states)
- hidden_states = attn_output + feed_forward_hidden_states + residual
-
- if use_cache:
- outputs = (hidden_states,) + outputs
- else:
- outputs = (hidden_states,) + outputs[1:]
-
- return outputs # hidden_states, present, (attentions)
-
-
-class MossPreTrainedModel(PreTrainedModel):
- """
- An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
- models.
- """
-
- config_class = MossConfig
- base_model_prefix = "transformer"
- supports_gradient_checkpointing = True
- _no_split_modules = ["MossBlock"]
-
- def __init__(self, *inputs, **kwargs):
- super().__init__(*inputs, **kwargs)
-
- def _init_weights(self, module):
- """Initialize the weights."""
- if isinstance(module, (nn.Linear,)):
- # Slightly different from Mesh Transformer JAX which uses truncated_normal for initialization
- # cf https://github.com/pytorch/pytorch/pull/5617
- module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
- if module.bias is not None:
- module.bias.data.zero_()
- elif isinstance(module, nn.Embedding):
- module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
- if module.padding_idx is not None:
- module.weight.data[module.padding_idx].zero_()
- elif isinstance(module, nn.LayerNorm):
- module.bias.data.zero_()
- module.weight.data.fill_(1.0)
-
- def _set_gradient_checkpointing(self, module, value=False):
- if isinstance(module, MossModel):
- module.gradient_checkpointing = value
-
-
-MOSS_START_DOCSTRING = r"""
- This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
- it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
- behavior.
-
- Parameters:
- config ([`MossConfig`]): Model configuration class with all the parameters of the model.
- Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
-"""
-
-MOSS_INPUTS_DOCSTRING = r"""
- Args:
- input_ids (`torch.LongTensor` of shape `({0})`):
- Indices of input sequence tokens in the vocabulary.
-
- Indices can be obtained using [`AutoProcenizer`]. See [`PreTrainedTokenizer.encode`] and
- [`PreTrainedTokenizer.__call__`] for details.
-
- [What are input IDs?](../glossary#input-ids)
- attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
- Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
-
- - 1 for tokens that are **not masked**,
- - 0 for tokens that are **masked**.
-
- [What are attention masks?](../glossary#attention-mask)
- token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
- 1]`:
-
- - 0 corresponds to a *sentence A* token,
- - 1 corresponds to a *sentence B* token.
-
- [What are token type IDs?](../glossary#token-type-ids)
- position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
- config.n_positions - 1]`.
-
- [What are position IDs?](../glossary#position-ids)
- head_mask (`torch.FloatTensor` of shape `(num_attention_heads,)` or `(n_layer, num_attention_heads)`, *optional*):
- Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:
-
- - 1 indicates the head is **not masked**,
- - 0 indicates the head is **masked**.
-
- inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_dim)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
- is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
- model's internal embedding lookup matrix.
- output_attentions (`bool`, *optional*):
- Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
- tensors for more detail.
- output_hidden_states (`bool`, *optional*):
- Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
- more detail.
- return_dict (`bool`, *optional*):
- Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
-"""
-
-
-@add_start_docstrings(
- "The bare Moss Model transformer outputting raw hidden-states without any specific head on top.",
- MOSS_START_DOCSTRING,
-)
-class MossModel(MossPreTrainedModel):
- def __init__(self, config):
- super().__init__(config)
-
- self.embed_dim = config.n_embd
- self.vocab_size = config.vocab_size
- self.wte = nn.Embedding(config.vocab_size, self.embed_dim)
- self.drop = nn.Dropout(config.embd_pdrop)
- self.h = nn.ModuleList([MossBlock(config) for _ in range(config.n_layer)])
- self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
- self.rotary_dim = min(config.rotary_dim, config.n_ctx // config.num_attention_heads)
-
- self.gradient_checkpointing = False
-
- # Initialize weights and apply final processing
- self.post_init()
-
- def get_input_embeddings(self):
- return self.wte
-
- def set_input_embeddings(self, new_embeddings):
- self.wte = new_embeddings
-
- @add_start_docstrings_to_model_forward(MOSS_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
- @add_code_sample_docstrings(
- checkpoint=_CHECKPOINT_FOR_DOC,
- output_type=BaseModelOutputWithPast,
- config_class=_CONFIG_FOR_DOC,
- )
- def forward(
- self,
- input_ids: Optional[torch.LongTensor] = None,
- past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
- attention_mask: Optional[torch.FloatTensor] = None,
- token_type_ids: Optional[torch.LongTensor] = None,
- position_ids: Optional[torch.LongTensor] = None,
- head_mask: Optional[torch.FloatTensor] = None,
- inputs_embeds: Optional[torch.FloatTensor] = None,
- use_cache: Optional[bool] = None,
- output_attentions: Optional[bool] = None,
- output_hidden_states: Optional[bool] = None,
- return_dict: Optional[bool] = None,
- ) -> Union[Tuple, BaseModelOutputWithPast]:
- output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
- output_hidden_states = (
- output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
- )
- use_cache = use_cache if use_cache is not None else self.config.use_cache
- return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-
- if input_ids is not None and inputs_embeds is not None:
- raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
- elif input_ids is not None:
- input_shape = input_ids.size()
- input_ids = input_ids.view(-1, input_shape[-1])
- batch_size = input_ids.shape[0]
- elif inputs_embeds is not None:
- input_shape = inputs_embeds.size()[:-1]
- batch_size = inputs_embeds.shape[0]
- else:
- raise ValueError("You have to specify either input_ids or inputs_embeds")
-
- device = input_ids.device if input_ids is not None else inputs_embeds.device
-
- if token_type_ids is not None:
- token_type_ids = token_type_ids.view(-1, input_shape[-1])
-
- if position_ids is not None:
- position_ids = position_ids.view(-1, input_shape[-1]).long()
-
- if past_key_values is None:
- past_length = 0
- past_key_values = tuple([None] * len(self.h))
- else:
- past_length = past_key_values[0][0].size(-2)
-
- if position_ids is None:
- position_ids = torch.arange(past_length, input_shape[-1] + past_length, dtype=torch.long, device=device)
- position_ids = position_ids.unsqueeze(0).view(-1, input_shape[-1])
-
- # Attention mask.
- if attention_mask is not None:
- if batch_size <= 0:
- raise ValueError("batch_size has to be defined and > 0")
- attention_mask = attention_mask.view(batch_size, -1)
- # We create a 3D attention mask from a 2D tensor mask.
- # Sizes are [batch_size, 1, 1, to_seq_length]
- # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length]
- # this attention mask is more simple than the triangular masking of causal attention
- # used in OpenAI GPT, we just need to prepare the broadcast dimension here.
- attention_mask = attention_mask[:, None, None, :]
-
- # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
- # masked positions, this operation will create a tensor which is 0.0 for
- # positions we want to attend and the dtype's smallest value for masked positions.
- # Since we are adding it to the raw scores before the softmax, this is
- # effectively the same as removing these entirely.
- attention_mask = attention_mask.to(dtype=self.dtype) # fp16 compatibility
- attention_mask = (1.0 - attention_mask) * torch.finfo(self.dtype).min
-
- # Prepare head mask if needed
- # 1.0 in head_mask indicate we keep the head
- # attention_probs has shape bsz x num_attention_heads x N x N
- # head_mask has shape n_layer x batch x num_attention_heads x N x N
- head_mask = self.get_head_mask(head_mask, self.config.n_layer)
-
- if inputs_embeds is None:
- inputs_embeds = self.wte(input_ids)
-
- hidden_states = inputs_embeds
-
- if token_type_ids is not None:
- token_type_embeds = self.wte(token_type_ids)
- hidden_states = hidden_states + token_type_embeds
-
- hidden_states = self.drop(hidden_states)
-
- output_shape = input_shape + (hidden_states.size(-1),)
-
- if self.gradient_checkpointing and self.training:
- if use_cache:
- logger.warning_once(
- "`use_cache=True` is incompatible with `config.gradient_checkpointing=True`. Setting "
- "`use_cache=False`..."
- )
- use_cache = False
-
- presents = () if use_cache else None
- all_self_attentions = () if output_attentions else None
- all_hidden_states = () if output_hidden_states else None
- for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)):
- if output_hidden_states:
- all_hidden_states = all_hidden_states + (hidden_states,)
-
- if self.gradient_checkpointing and self.training:
-
- def create_custom_forward(module):
- def custom_forward(*inputs):
- # None for past_key_value
- return module(*inputs, use_cache, output_attentions)
-
- return custom_forward
-
- outputs = torch.utils.checkpoint.checkpoint(
- create_custom_forward(block),
- hidden_states,
- None,
- attention_mask,
- position_ids,
- head_mask[i],
- )
- else:
- outputs = block(
- hidden_states=hidden_states,
- layer_past=layer_past,
- attention_mask=attention_mask,
- position_ids=position_ids,
- head_mask=head_mask[i],
- use_cache=use_cache,
- output_attentions=output_attentions,
- )
-
- hidden_states = outputs[0]
- if use_cache is True:
- presents = presents + (outputs[1],)
-
- if output_attentions:
- all_self_attentions = all_self_attentions + (outputs[2 if use_cache else 1],)
-
- hidden_states = self.ln_f(hidden_states)
-
- hidden_states = hidden_states.view(output_shape)
- # Add last hidden state
- if output_hidden_states:
- all_hidden_states = all_hidden_states + (hidden_states,)
-
- if not return_dict:
- return tuple(v for v in [hidden_states, presents, all_hidden_states, all_self_attentions] if v is not None)
-
- return BaseModelOutputWithPast(
- last_hidden_state=hidden_states,
- past_key_values=presents,
- hidden_states=all_hidden_states,
- attentions=all_self_attentions,
- )
-
-
-@add_start_docstrings(
- """
- The Moss Model transformer with a language modeling head on top.
- """,
- MOSS_START_DOCSTRING,
-)
-class MossForCausalLM(MossPreTrainedModel):
- _keys_to_ignore_on_load_missing = [r"h\.\d+\.attn\.causal_mask"]
-
- def __init__(self, config):
- super().__init__(config)
- if not hasattr(config, 'wbits'):
- config.wbits = 32
- config.groupsize = 128
-
- if config.wbits not in [4, 8, 32]:
- logger.warning(f'Specify `wbits` with 4, 8 or 32 to load the model. ')
- if config.wbits in [4, 8]:
- def noop(*args, **kwargs):
- pass
- torch.nn.init.kaiming_uniform_ = noop
- torch.nn.init.uniform_ = noop
- torch.nn.init.normal_ = noop
-
- torch.set_default_dtype(torch.half)
- transformers.modeling_utils._init_weights = False
- torch.set_default_dtype(torch.half)
- self.transformer = MossModel(config)
- self.lm_head = nn.Linear(config.n_embd, config.vocab_size)
- if config.wbits in [4, 8]:
- torch.set_default_dtype(torch.float)
- transformers.modeling_utils._init_weights = True
- self.quantize(config.wbits, config.groupsize)
- # Initialize weights and apply final processing
- self.post_init()
-
- def get_output_embeddings(self):
- return self.lm_head
-
- def set_output_embeddings(self, new_embeddings):
- self.lm_head = new_embeddings
-
- def prepare_inputs_for_generation(self, input_ids, past_key_values=None, **kwargs):
- token_type_ids = kwargs.get("token_type_ids", None)
- # only last token for inputs_ids if past is defined in kwargs
- if past_key_values:
- input_ids = input_ids[:, -1].unsqueeze(-1)
- if token_type_ids is not None:
- token_type_ids = token_type_ids[:, -1].unsqueeze(-1)
-
- attention_mask = kwargs.get("attention_mask", None)
- position_ids = kwargs.get("position_ids", None)
-
- if attention_mask is not None and position_ids is None:
- # create position_ids on the fly for batch generation
- position_ids = attention_mask.long().cumsum(-1) - 1
- position_ids.masked_fill_(attention_mask == 0, 1)
- if past_key_values:
- position_ids = position_ids[:, -1].unsqueeze(-1)
-
- return {
- "input_ids": input_ids,
- "past_key_values": past_key_values,
- "use_cache": kwargs.get("use_cache"),
- "position_ids": position_ids,
- "attention_mask": attention_mask,
- "token_type_ids": token_type_ids,
- }
-
- @add_start_docstrings_to_model_forward(MOSS_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
- @add_code_sample_docstrings(
- checkpoint=_CHECKPOINT_FOR_DOC,
- output_type=CausalLMOutputWithPast,
- config_class=_CONFIG_FOR_DOC,
- )
- def forward(
- self,
- input_ids: Optional[torch.LongTensor] = None,
- past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
- attention_mask: Optional[torch.FloatTensor] = None,
- token_type_ids: Optional[torch.LongTensor] = None,
- position_ids: Optional[torch.LongTensor] = None,
- head_mask: Optional[torch.FloatTensor] = None,
- inputs_embeds: Optional[torch.FloatTensor] = None,
- labels: Optional[torch.LongTensor] = None,
- use_cache: Optional[bool] = None,
- output_attentions: Optional[bool] = None,
- output_hidden_states: Optional[bool] = None,
- return_dict: Optional[bool] = None,
- ) -> Union[Tuple, CausalLMOutputWithPast]:
- r"""
- labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
- `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
- are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
- """
- return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-
- transformer_outputs = self.transformer(
- input_ids,
- past_key_values=past_key_values,
- attention_mask=attention_mask,
- token_type_ids=token_type_ids,
- position_ids=position_ids,
- head_mask=head_mask,
- inputs_embeds=inputs_embeds,
- use_cache=use_cache,
- output_attentions=output_attentions,
- output_hidden_states=output_hidden_states,
- return_dict=return_dict,
- )
- hidden_states = transformer_outputs[0]
-
- # make sure sampling in fp16 works correctly and
- # compute loss in fp32 to match with mesh-tf version
- # https://github.com/EleutherAI/gpt-neo/blob/89ce74164da2fb16179106f54e2269b5da8db333/models/gpt2/gpt2.py#L179
- lm_logits = self.lm_head(hidden_states).to(torch.float32)
-
- loss = None
- if labels is not None:
- # Shift so that tokens < n predict n
- shift_logits = lm_logits[..., :-1, :].contiguous()
- shift_labels = labels[..., 1:].contiguous()
- # Flatten the tokens
- loss_fct = CrossEntropyLoss()
- loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
-
- loss = loss.to(hidden_states.dtype)
-
- if not return_dict:
- output = (lm_logits,) + transformer_outputs[1:]
- return ((loss,) + output) if loss is not None else output
-
- return CausalLMOutputWithPast(
- loss=loss,
- logits=lm_logits,
- past_key_values=transformer_outputs.past_key_values,
- hidden_states=transformer_outputs.hidden_states,
- attentions=transformer_outputs.attentions,
- )
-
- @staticmethod
- def _reorder_cache(
- past_key_values: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor
- ) -> Tuple[Tuple[torch.Tensor]]:
- """
- This function is used to re-order the `past_key_values` cache if [`~PretrainedModel.beam_search`] or
- [`~PretrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct
- beam_idx at every generation step.
- """
- return tuple(
- tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past)
- for layer_past in past_key_values
- )
-
- def quantize(self, wbits, groupsize):
- from .quantization import quantize_with_gptq
- return quantize_with_gptq(self, wbits, groupsize)
-
diff --git a/multi_language.py b/multi_language.py
index c65872aa08a0b0088d94467054594c422978ca38..8e3ac9d7208d4258be408e8ddc3126baf98c6967 100644
--- a/multi_language.py
+++ b/multi_language.py
@@ -13,7 +13,6 @@
4. Run `python multi_language.py`.
Note: You need to run it multiple times to increase translation coverage because GPT makes mistakes sometimes.
- (You can also run `CACHE_ONLY=True python multi_language.py` to use cached translation mapping)
5. Find the translated program in `multi-language\English\*`
@@ -36,9 +35,7 @@ import pickle
import time
from toolbox import get_conf
-CACHE_ONLY = os.environ.get('CACHE_ONLY', False)
-
-CACHE_FOLDER = get_conf('PATH_LOGGING')
+CACHE_FOLDER, = get_conf('PATH_LOGGING')
blacklist = ['multi-language', CACHE_FOLDER, '.git', 'private_upload', 'multi_language.py', 'build', '.github', '.vscode', '__pycache__', 'venv']
@@ -182,12 +179,12 @@ cached_translation = read_map_from_json(language=LANG)
def trans(word_to_translate, language, special=False):
if len(word_to_translate) == 0: return {}
from crazy_functions.crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency
- from toolbox import get_conf, ChatBotWithCookies, load_chat_cookies
-
- cookies = load_chat_cookies()
+ from toolbox import get_conf, ChatBotWithCookies
+ proxies, WEB_PORT, LLM_MODEL, CONCURRENT_COUNT, AUTHENTICATION, CHATBOT_HEIGHT, LAYOUT, API_KEY = \
+ get_conf('proxies', 'WEB_PORT', 'LLM_MODEL', 'CONCURRENT_COUNT', 'AUTHENTICATION', 'CHATBOT_HEIGHT', 'LAYOUT', 'API_KEY')
llm_kwargs = {
- 'api_key': cookies['api_key'],
- 'llm_model': cookies['llm_model'],
+ 'api_key': API_KEY,
+ 'llm_model': LLM_MODEL,
'top_p':1.0,
'max_length': None,
'temperature':0.4,
@@ -245,15 +242,15 @@ def trans(word_to_translate, language, special=False):
def trans_json(word_to_translate, language, special=False):
if len(word_to_translate) == 0: return {}
from crazy_functions.crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency
- from toolbox import get_conf, ChatBotWithCookies, load_chat_cookies
-
- cookies = load_chat_cookies()
+ from toolbox import get_conf, ChatBotWithCookies
+ proxies, WEB_PORT, LLM_MODEL, CONCURRENT_COUNT, AUTHENTICATION, CHATBOT_HEIGHT, LAYOUT, API_KEY = \
+ get_conf('proxies', 'WEB_PORT', 'LLM_MODEL', 'CONCURRENT_COUNT', 'AUTHENTICATION', 'CHATBOT_HEIGHT', 'LAYOUT', 'API_KEY')
llm_kwargs = {
- 'api_key': cookies['api_key'],
- 'llm_model': cookies['llm_model'],
+ 'api_key': API_KEY,
+ 'llm_model': LLM_MODEL,
'top_p':1.0,
'max_length': None,
- 'temperature':0.4,
+ 'temperature':0.1,
}
import random
N_EACH_REQ = random.randint(16, 32)
@@ -339,10 +336,7 @@ def step_1_core_key_translate():
if d not in cached_translation_keys:
need_translate.append(d)
- if CACHE_ONLY:
- need_translate_mapping = {}
- else:
- need_translate_mapping = trans(need_translate, language=LANG_STD, special=True)
+ need_translate_mapping = trans(need_translate, language=LANG_STD, special=True)
map_to_json(need_translate_mapping, language=LANG_STD)
cached_translation = read_map_from_json(language=LANG_STD)
cached_translation = dict(sorted(cached_translation.items(), key=lambda x: -len(x[0])))
@@ -352,9 +346,9 @@ def step_1_core_key_translate():
chinese_core_keys_norepeat_mapping.update({k:cached_translation[k]})
chinese_core_keys_norepeat_mapping = dict(sorted(chinese_core_keys_norepeat_mapping.items(), key=lambda x: -len(x[0])))
- # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+ # ===============================================
# copy
- # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+ # ===============================================
def copy_source_code():
from toolbox import get_conf
@@ -367,9 +361,9 @@ def step_1_core_key_translate():
shutil.copytree('./', backup_dir, ignore=lambda x, y: blacklist)
copy_source_code()
- # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+ # ===============================================
# primary key replace
- # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+ # ===============================================
directory_path = f'./multi-language/{LANG}/'
for root, dirs, files in os.walk(directory_path):
for file in files:
@@ -389,9 +383,9 @@ def step_1_core_key_translate():
def step_2_core_key_translate():
- # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
+ # =================================================================================================
# step2
- # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
+ # =================================================================================================
def load_string(strings, string_input):
string_ = string_input.strip().strip(',').strip().strip('.').strip()
@@ -482,19 +476,17 @@ def step_2_core_key_translate():
if d not in cached_translation_keys:
need_translate.append(d)
- if CACHE_ONLY:
- up = {}
- else:
- up = trans_json(need_translate, language=LANG, special=False)
+
+ up = trans_json(need_translate, language=LANG, special=False)
map_to_json(up, language=LANG)
cached_translation = read_map_from_json(language=LANG)
LANG_STD = 'std'
cached_translation.update(read_map_from_json(language=LANG_STD))
cached_translation = dict(sorted(cached_translation.items(), key=lambda x: -len(x[0])))
- # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+ # ===============================================
# literal key replace
- # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+ # ===============================================
directory_path = f'./multi-language/{LANG}/'
for root, dirs, files in os.walk(directory_path):
for file in files:
diff --git a/request_llm/README.md b/request_llm/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..545bc1ffba8b79a49d994cfedcc2a787475181b2
--- /dev/null
+++ b/request_llm/README.md
@@ -0,0 +1,79 @@
+# 如何使用其他大语言模型
+
+## ChatGLM
+
+- 安装依赖 `pip install -r request_llm/requirements_chatglm.txt`
+- 修改配置，在config.py中将LLM_MODEL的值改为"chatglm"
+
+``` sh
+LLM_MODEL = "chatglm"
+```
+- 运行！
+``` sh
+`python main.py`
+```
+
+## Claude-Stack
+
+- 请参考此教程获取 https://zhuanlan.zhihu.com/p/627485689
+ - 1、SLACK_CLAUDE_BOT_ID
+ - 2、SLACK_CLAUDE_USER_TOKEN
+
+- 把token加入config.py
+
+## Newbing
+
+- 使用cookie editor获取cookie（json）
+- 把cookie（json）加入config.py （NEWBING_COOKIES）
+
+## Moss
+- 使用docker-compose
+
+## RWKV
+- 使用docker-compose
+
+## LLAMA
+- 使用docker-compose
+
+## 盘古
+- 使用docker-compose
+
+
+---
+## Text-Generation-UI (TGUI，调试中，暂不可用)
+
+### 1. 部署TGUI
+``` sh
+# 1 下载模型
+git clone https://github.com/oobabooga/text-generation-webui.git
+# 2 这个仓库的最新代码有问题，回滚到几周之前
+git reset --hard fcda3f87767e642d1c0411776e549e1d3894843d
+# 3 切换路径
+cd text-generation-webui
+# 4 安装text-generation的额外依赖
+pip install accelerate bitsandbytes flexgen gradio llamacpp markdown numpy peft requests rwkv safetensors sentencepiece tqdm datasets git+https://github.com/huggingface/transformers
+# 5 下载模型
+python download-model.py facebook/galactica-1.3b
+# 其他可选如 facebook/opt-1.3b
+# facebook/galactica-1.3b
+# facebook/galactica-6.7b
+# facebook/galactica-120b
+# facebook/pygmalion-1.3b 等
+# 详情见 https://github.com/oobabooga/text-generation-webui
+
+# 6 启动text-generation
+python server.py --cpu --listen --listen-port 7865 --model facebook_galactica-1.3b
+```
+
+### 2. 修改config.py
+
+``` sh
+# LLM_MODEL格式: tgui:[模型]@[ws地址]:[ws端口] , 端口要和上面给定的端口一致
+LLM_MODEL = "tgui:galactica-1.3b@localhost:7860"
+```
+
+### 3. 运行！
+``` sh
+cd chatgpt-academic
+python main.py
+```
diff --git a/request_llms/bridge_all.py b/request_llm/bridge_all.py
similarity index 65%
rename from request_llms/bridge_all.py
rename to request_llm/bridge_all.py
index e20570f93502c5a6dc633640c90f5968f7c70f1f..bb325e460742cececeaf1683d331c593bcba2915 100644
--- a/request_llms/bridge_all.py
+++ b/request_llm/bridge_all.py
@@ -8,32 +8,23 @@
具备多线程调用能力的函数：在函数插件中被调用，灵活而简洁
2. predict_no_ui_long_connection(...)
"""
-import tiktoken, copy
+import tiktoken
from functools import lru_cache
from concurrent.futures import ThreadPoolExecutor
-from toolbox import get_conf, trimmed_format_exc, apply_gpt_academic_string_mask
+from toolbox import get_conf, trimmed_format_exc
from .bridge_chatgpt import predict_no_ui_long_connection as chatgpt_noui
from .bridge_chatgpt import predict as chatgpt_ui
-from .bridge_chatgpt_vision import predict_no_ui_long_connection as chatgpt_vision_noui
-from .bridge_chatgpt_vision import predict as chatgpt_vision_ui
-
from .bridge_chatglm import predict_no_ui_long_connection as chatglm_noui
from .bridge_chatglm import predict as chatglm_ui
-from .bridge_chatglm3 import predict_no_ui_long_connection as chatglm3_noui
-from .bridge_chatglm3 import predict as chatglm3_ui
+from .bridge_chatglm import predict_no_ui_long_connection as chatglm_noui
+from .bridge_chatglm import predict as chatglm_ui
from .bridge_qianfan import predict_no_ui_long_connection as qianfan_noui
from .bridge_qianfan import predict as qianfan_ui
-from .bridge_google_gemini import predict as genai_ui
-from .bridge_google_gemini import predict_no_ui_long_connection as genai_noui
-
-from .bridge_zhipu import predict_no_ui_long_connection as zhipu_noui
-from .bridge_zhipu import predict as zhipu_ui
-
colors = ['#FF00FF', '#00FFFF', '#FF0000', '#990099', '#009999', '#990044']
class LazyloadTiktoken(object):
@@ -47,13 +38,13 @@ class LazyloadTiktoken(object):
tmp = tiktoken.encoding_for_model(model)
print('加载tokenizer完毕')
return tmp
-
+
def encode(self, *args, **kwargs):
- encoder = self.get_encoder(self.model)
+ encoder = self.get_encoder(self.model)
return encoder.encode(*args, **kwargs)
-
+
def decode(self, *args, **kwargs):
- encoder = self.get_encoder(self.model)
+ encoder = self.get_encoder(self.model)
return encoder.decode(*args, **kwargs)
# Endpoint 重定向
@@ -61,12 +52,11 @@ API_URL_REDIRECT, AZURE_ENDPOINT, AZURE_ENGINE = get_conf("API_URL_REDIRECT", "A
openai_endpoint = "https://api.openai.com/v1/chat/completions"
api2d_endpoint = "https://openai.api2d.net/v1/chat/completions"
newbing_endpoint = "wss://sydney.bing.com/sydney/ChatHub"
-if not AZURE_ENDPOINT.endswith('/'): AZURE_ENDPOINT += '/'
azure_endpoint = AZURE_ENDPOINT + f'openai/deployments/{AZURE_ENGINE}/chat/completions?api-version=2023-05-15'
# 兼容旧版的配置
try:
- API_URL = get_conf("API_URL")
- if API_URL != "https://api.openai.com/v1/chat/completions":
+ API_URL, = get_conf("API_URL")
+ if API_URL != "https://api.openai.com/v1/chat/completions":
openai_endpoint = API_URL
print("警告！API_URL配置选项将被弃用，请更换为API_URL_REDIRECT配置")
except:
@@ -98,12 +88,12 @@ model_info = {
"tokenizer": tokenizer_gpt35,
"token_cnt": get_token_num_gpt35,
},
-
+
"gpt-3.5-turbo-16k": {
"fn_with_ui": chatgpt_ui,
"fn_without_ui": chatgpt_noui,
"endpoint": openai_endpoint,
- "max_token": 16385,
+ "max_token": 1024*16,
"tokenizer": tokenizer_gpt35,
"token_cnt": get_token_num_gpt35,
},
@@ -121,16 +111,7 @@ model_info = {
"fn_with_ui": chatgpt_ui,
"fn_without_ui": chatgpt_noui,
"endpoint": openai_endpoint,
- "max_token": 16385,
- "tokenizer": tokenizer_gpt35,
- "token_cnt": get_token_num_gpt35,
- },
-
- "gpt-3.5-turbo-1106": {#16k
- "fn_with_ui": chatgpt_ui,
- "fn_without_ui": chatgpt_noui,
- "endpoint": openai_endpoint,
- "max_token": 16385,
+ "max_token": 1024 * 16,
"tokenizer": tokenizer_gpt35,
"token_cnt": get_token_num_gpt35,
},
@@ -144,61 +125,6 @@ model_info = {
"token_cnt": get_token_num_gpt4,
},
- "gpt-4-32k": {
- "fn_with_ui": chatgpt_ui,
- "fn_without_ui": chatgpt_noui,
- "endpoint": openai_endpoint,
- "max_token": 32768,
- "tokenizer": tokenizer_gpt4,
- "token_cnt": get_token_num_gpt4,
- },
-
- "gpt-4-turbo-preview": {
- "fn_with_ui": chatgpt_ui,
- "fn_without_ui": chatgpt_noui,
- "endpoint": openai_endpoint,
- "max_token": 128000,
- "tokenizer": tokenizer_gpt4,
- "token_cnt": get_token_num_gpt4,
- },
-
- "gpt-4-1106-preview": {
- "fn_with_ui": chatgpt_ui,
- "fn_without_ui": chatgpt_noui,
- "endpoint": openai_endpoint,
- "max_token": 128000,
- "tokenizer": tokenizer_gpt4,
- "token_cnt": get_token_num_gpt4,
- },
-
- "gpt-4-0125-preview": {
- "fn_with_ui": chatgpt_ui,
- "fn_without_ui": chatgpt_noui,
- "endpoint": openai_endpoint,
- "max_token": 128000,
- "tokenizer": tokenizer_gpt4,
- "token_cnt": get_token_num_gpt4,
- },
-
- "gpt-3.5-random": {
- "fn_with_ui": chatgpt_ui,
- "fn_without_ui": chatgpt_noui,
- "endpoint": openai_endpoint,
- "max_token": 4096,
- "tokenizer": tokenizer_gpt4,
- "token_cnt": get_token_num_gpt4,
- },
-
- "gpt-4-vision-preview": {
- "fn_with_ui": chatgpt_vision_ui,
- "fn_without_ui": chatgpt_vision_noui,
- "endpoint": openai_endpoint,
- "max_token": 4096,
- "tokenizer": tokenizer_gpt4,
- "token_cnt": get_token_num_gpt4,
- },
-
-
# azure openai
"azure-gpt-3.5":{
"fn_with_ui": chatgpt_ui,
@@ -209,34 +135,16 @@ model_info = {
"token_cnt": get_token_num_gpt35,
},
- "azure-gpt-4":{
+ # api_2d
+ "api2d-gpt-3.5-turbo": {
"fn_with_ui": chatgpt_ui,
"fn_without_ui": chatgpt_noui,
- "endpoint": azure_endpoint,
- "max_token": 8192,
- "tokenizer": tokenizer_gpt4,
- "token_cnt": get_token_num_gpt4,
- },
-
- # 智谱AI
- "glm-4": {
- "fn_with_ui": zhipu_ui,
- "fn_without_ui": zhipu_noui,
- "endpoint": None,
- "max_token": 10124 * 8,
- "tokenizer": tokenizer_gpt35,
- "token_cnt": get_token_num_gpt35,
- },
- "glm-3-turbo": {
- "fn_with_ui": zhipu_ui,
- "fn_without_ui": zhipu_noui,
- "endpoint": None,
- "max_token": 10124 * 4,
+ "endpoint": api2d_endpoint,
+ "max_token": 4096,
"tokenizer": tokenizer_gpt35,
"token_cnt": get_token_num_gpt35,
},
- # api_2d (此后不需要在此处添加api2d的接口了，因为下面的代码会自动添加)
"api2d-gpt-4": {
"fn_with_ui": chatgpt_ui,
"fn_without_ui": chatgpt_noui,
@@ -263,14 +171,6 @@ model_info = {
"tokenizer": tokenizer_gpt35,
"token_cnt": get_token_num_gpt35,
},
- "chatglm3": {
- "fn_with_ui": chatglm3_ui,
- "fn_without_ui": chatglm3_noui,
- "endpoint": None,
- "max_token": 8192,
- "tokenizer": tokenizer_gpt35,
- "token_cnt": get_token_num_gpt35,
- },
"qianfan": {
"fn_with_ui": qianfan_ui,
"fn_without_ui": qianfan_noui,
@@ -279,38 +179,8 @@ model_info = {
"tokenizer": tokenizer_gpt35,
"token_cnt": get_token_num_gpt35,
},
- "gemini-pro": {
- "fn_with_ui": genai_ui,
- "fn_without_ui": genai_noui,
- "endpoint": None,
- "max_token": 1024 * 32,
- "tokenizer": tokenizer_gpt35,
- "token_cnt": get_token_num_gpt35,
- },
- "gemini-pro-vision": {
- "fn_with_ui": genai_ui,
- "fn_without_ui": genai_noui,
- "endpoint": None,
- "max_token": 1024 * 32,
- "tokenizer": tokenizer_gpt35,
- "token_cnt": get_token_num_gpt35,
- },
}
-# -=-=-=-=-=-=- api2d 对齐支持 -=-=-=-=-=-=-
-for model in AVAIL_LLM_MODELS:
- if model.startswith('api2d-') and (model.replace('api2d-','') in model_info.keys()):
- mi = copy.deepcopy(model_info[model.replace('api2d-','')])
- mi.update({"endpoint": api2d_endpoint})
- model_info.update({model: mi})
-
-# -=-=-=-=-=-=- azure 对齐支持 -=-=-=-=-=-=-
-for model in AVAIL_LLM_MODELS:
- if model.startswith('azure-') and (model.replace('azure-','') in model_info.keys()):
- mi = copy.deepcopy(model_info[model.replace('azure-','')])
- mi.update({"endpoint": azure_endpoint})
- model_info.update({model: mi})
-
# -=-=-=-=-=-=- 以下部分是新加入的模型，可能附带额外依赖 -=-=-=-=-=-=-
if "claude-1-100k" in AVAIL_LLM_MODELS or "claude-2" in AVAIL_LLM_MODELS:
from .bridge_claude import predict_no_ui_long_connection as claude_noui
@@ -480,63 +350,31 @@ if "chatglm_onnx" in AVAIL_LLM_MODELS:
})
except:
print(trimmed_format_exc())
-if "qwen-local" in AVAIL_LLM_MODELS:
- try:
- from .bridge_qwen_local import predict_no_ui_long_connection as qwen_local_noui
- from .bridge_qwen_local import predict as qwen_local_ui
- model_info.update({
- "qwen-local": {
- "fn_with_ui": qwen_local_ui,
- "fn_without_ui": qwen_local_noui,
- "endpoint": None,
- "max_token": 4096,
- "tokenizer": tokenizer_gpt35,
- "token_cnt": get_token_num_gpt35,
- }
- })
- except:
- print(trimmed_format_exc())
-if "qwen-turbo" in AVAIL_LLM_MODELS or "qwen-plus" in AVAIL_LLM_MODELS or "qwen-max" in AVAIL_LLM_MODELS: # zhipuai
+if "qwen" in AVAIL_LLM_MODELS:
try:
from .bridge_qwen import predict_no_ui_long_connection as qwen_noui
from .bridge_qwen import predict as qwen_ui
model_info.update({
- "qwen-turbo": {
- "fn_with_ui": qwen_ui,
- "fn_without_ui": qwen_noui,
- "endpoint": None,
- "max_token": 6144,
- "tokenizer": tokenizer_gpt35,
- "token_cnt": get_token_num_gpt35,
- },
- "qwen-plus": {
+ "qwen": {
"fn_with_ui": qwen_ui,
"fn_without_ui": qwen_noui,
"endpoint": None,
- "max_token": 30720,
- "tokenizer": tokenizer_gpt35,
- "token_cnt": get_token_num_gpt35,
- },
- "qwen-max": {
- "fn_with_ui": qwen_ui,
- "fn_without_ui": qwen_noui,
- "endpoint": None,
- "max_token": 28672,
+ "max_token": 4096,
"tokenizer": tokenizer_gpt35,
"token_cnt": get_token_num_gpt35,
}
})
except:
print(trimmed_format_exc())
-if "spark" in AVAIL_LLM_MODELS: # 讯飞星火认知大模型
+if "chatgpt_website" in AVAIL_LLM_MODELS: # 接入一些逆向工程https://github.com/acheong08/ChatGPT-to-API/
try:
- from .bridge_spark import predict_no_ui_long_connection as spark_noui
- from .bridge_spark import predict as spark_ui
+ from .bridge_chatgpt_website import predict_no_ui_long_connection as chatgpt_website_noui
+ from .bridge_chatgpt_website import predict as chatgpt_website_ui
model_info.update({
- "spark": {
- "fn_with_ui": spark_ui,
- "fn_without_ui": spark_noui,
- "endpoint": None,
+ "chatgpt_website": {
+ "fn_with_ui": chatgpt_website_ui,
+ "fn_without_ui": chatgpt_website_noui,
+ "endpoint": openai_endpoint,
"max_token": 4096,
"tokenizer": tokenizer_gpt35,
"token_cnt": get_token_num_gpt35,
@@ -544,12 +382,12 @@ if "spark" in AVAIL_LLM_MODELS: # 讯飞星火认知大模型
})
except:
print(trimmed_format_exc())
-if "sparkv2" in AVAIL_LLM_MODELS: # 讯飞星火认知大模型
+if "spark" in AVAIL_LLM_MODELS: # 讯飞星火认知大模型
try:
from .bridge_spark import predict_no_ui_long_connection as spark_noui
from .bridge_spark import predict as spark_ui
model_info.update({
- "sparkv2": {
+ "spark": {
"fn_with_ui": spark_ui,
"fn_without_ui": spark_noui,
"endpoint": None,
@@ -560,20 +398,12 @@ if "sparkv2" in AVAIL_LLM_MODELS: # 讯飞星火认知大模型
})
except:
print(trimmed_format_exc())
-if "sparkv3" in AVAIL_LLM_MODELS or "sparkv3.5" in AVAIL_LLM_MODELS: # 讯飞星火认知大模型
+if "sparkv2" in AVAIL_LLM_MODELS: # 讯飞星火认知大模型
try:
from .bridge_spark import predict_no_ui_long_connection as spark_noui
from .bridge_spark import predict as spark_ui
model_info.update({
- "sparkv3": {
- "fn_with_ui": spark_ui,
- "fn_without_ui": spark_noui,
- "endpoint": None,
- "max_token": 4096,
- "tokenizer": tokenizer_gpt35,
- "token_cnt": get_token_num_gpt35,
- },
- "sparkv3.5": {
+ "sparkv2": {
"fn_with_ui": spark_ui,
"fn_without_ui": spark_noui,
"endpoint": None,
@@ -600,77 +430,6 @@ if "llama2" in AVAIL_LLM_MODELS: # llama2
})
except:
print(trimmed_format_exc())
-if "zhipuai" in AVAIL_LLM_MODELS: # zhipuai 是glm-4的别名，向后兼容配置
- try:
- model_info.update({
- "zhipuai": {
- "fn_with_ui": zhipu_ui,
- "fn_without_ui": zhipu_noui,
- "endpoint": None,
- "max_token": 10124 * 8,
- "tokenizer": tokenizer_gpt35,
- "token_cnt": get_token_num_gpt35,
- },
- })
- except:
- print(trimmed_format_exc())
-if "deepseekcoder" in AVAIL_LLM_MODELS: # deepseekcoder
- try:
- from .bridge_deepseekcoder import predict_no_ui_long_connection as deepseekcoder_noui
- from .bridge_deepseekcoder import predict as deepseekcoder_ui
- model_info.update({
- "deepseekcoder": {
- "fn_with_ui": deepseekcoder_ui,
- "fn_without_ui": deepseekcoder_noui,
- "endpoint": None,
- "max_token": 2048,
- "tokenizer": tokenizer_gpt35,
- "token_cnt": get_token_num_gpt35,
- }
- })
- except:
- print(trimmed_format_exc())
-# if "skylark" in AVAIL_LLM_MODELS:
-# try:
-# from .bridge_skylark2 import predict_no_ui_long_connection as skylark_noui
-# from .bridge_skylark2 import predict as skylark_ui
-# model_info.update({
-# "skylark": {
-# "fn_with_ui": skylark_ui,
-# "fn_without_ui": skylark_noui,
-# "endpoint": None,
-# "max_token": 4096,
-# "tokenizer": tokenizer_gpt35,
-# "token_cnt": get_token_num_gpt35,
-# }
-# })
-# except:
-# print(trimmed_format_exc())
-
-
-# <-- 用于定义和切换多个azure模型 -->
-AZURE_CFG_ARRAY = get_conf("AZURE_CFG_ARRAY")
-if len(AZURE_CFG_ARRAY) > 0:
- for azure_model_name, azure_cfg_dict in AZURE_CFG_ARRAY.items():
- # 可能会覆盖之前的配置，但这是意料之中的
- if not azure_model_name.startswith('azure'):
- raise ValueError("AZURE_CFG_ARRAY中配置的模型必须以azure开头")
- endpoint_ = azure_cfg_dict["AZURE_ENDPOINT"] + \
- f'openai/deployments/{azure_cfg_dict["AZURE_ENGINE"]}/chat/completions?api-version=2023-05-15'
- model_info.update({
- azure_model_name: {
- "fn_with_ui": chatgpt_ui,
- "fn_without_ui": chatgpt_noui,
- "endpoint": endpoint_,
- "azure_api_key": azure_cfg_dict["AZURE_API_KEY"],
- "max_token": azure_cfg_dict["AZURE_MODEL_MAX_TOKEN"],
- "tokenizer": tokenizer_gpt35, # tokenizer只用于粗估token数量
- "token_cnt": get_token_num_gpt35,
- }
- })
- if azure_model_name not in AVAIL_LLM_MODELS:
- AVAIL_LLM_MODELS += [azure_model_name]
-
@@ -688,7 +447,7 @@ def LLM_CATCH_EXCEPTION(f):
return decorated
-def predict_no_ui_long_connection(inputs, llm_kwargs, history, sys_prompt, observe_window=[], console_slience=False):
+def predict_no_ui_long_connection(inputs, llm_kwargs, history, sys_prompt, observe_window, console_slience=False):
"""
发送至LLM，等待回复，一次性完成，不显示中间过程。但内部用stream的方法避免中途网线被掐。
inputs：
@@ -704,7 +463,6 @@ def predict_no_ui_long_connection(inputs, llm_kwargs, history, sys_prompt, obser
"""
import threading, time, copy
- inputs = apply_gpt_academic_string_mask(inputs, mode="show_llm")
model = llm_kwargs['llm_model']
n_model = 1
if '&' not in model:
@@ -719,7 +477,7 @@ def predict_no_ui_long_connection(inputs, llm_kwargs, history, sys_prompt, obser
executor = ThreadPoolExecutor(max_workers=4)
models = model.split('&')
n_model = len(models)
-
+
window_len = len(observe_window)
assert window_len==3
window_mutex = [["", time.time(), ""] for _ in range(n_model)] + [True]
@@ -738,7 +496,7 @@ def predict_no_ui_long_connection(inputs, llm_kwargs, history, sys_prompt, obser
time.sleep(0.25)
if not window_mutex[-1]: break
# 看门狗（watchdog）
- for i in range(n_model):
+ for i in range(n_model):
window_mutex[i][1] = observe_window[1]
# 观察窗（window）
chat_string = []
@@ -778,7 +536,6 @@ def predict(inputs, llm_kwargs, *args, **kwargs):
additional_fn代表点击的哪个按钮，按钮见functional.py
"""
- inputs = apply_gpt_academic_string_mask(inputs, mode="show_llm")
method = model_info[llm_kwargs['llm_model']]["fn_with_ui"] # 如果这里报错，检查config中的AVAIL_LLM_MODELS选项
yield from method(inputs, llm_kwargs, *args, **kwargs)
diff --git a/request_llm/bridge_azure_test.py b/request_llm/bridge_azure_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..edc68f747d650e20a9e42d65dbcac1923d5cb192
--- /dev/null
+++ b/request_llm/bridge_azure_test.py
@@ -0,0 +1,241 @@
+"""
+ 该文件中主要包含三个函数
+
+ 不具备多线程能力的函数：
+ 1. predict: 正常对话时使用，具备完备的交互功能，不可多线程
+
+ 具备多线程调用能力的函数
+ 2. predict_no_ui：高级实验性功能模块调用，不会实时显示在界面上，参数简单，可以多线程并行，方便实现复杂的功能逻辑
+ 3. predict_no_ui_long_connection：在实验过程中发现调用predict_no_ui处理长文档时，和openai的连接容易断掉，这个函数用stream的方式解决这个问题，同样支持多线程
+"""
+
+import logging
+import traceback
+import importlib
+import openai
+import time
+
+
+# 读取config.py文件中关于AZURE OPENAI API的信息
+from toolbox import get_conf, update_ui, clip_history, trimmed_format_exc
+TIMEOUT_SECONDS, MAX_RETRY, AZURE_ENGINE, AZURE_ENDPOINT, AZURE_API_VERSION, AZURE_API_KEY = \
+ get_conf('TIMEOUT_SECONDS', 'MAX_RETRY',"AZURE_ENGINE","AZURE_ENDPOINT", "AZURE_API_VERSION", "AZURE_API_KEY")
+
+
+def get_full_error(chunk, stream_response):
+ """
+ 获取完整的从Openai返回的报错
+ """
+ while True:
+ try:
+ chunk += next(stream_response)
+ except:
+ break
+ return chunk
+
+def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_prompt='', stream = True, additional_fn=None):
+ """
+ 发送至azure openai api，流式获取输出。
+ 用于基础的对话功能。
+ inputs 是本次问询的输入
+ top_p, temperature是chatGPT的内部调优参数
+ history 是之前的对话列表（注意无论是inputs还是history，内容太长了都会触发token数量溢出的错误）
+ chatbot 为WebUI中显示的对话列表，修改它，然后yeild出去，可以直接修改对话界面内容
+ additional_fn代表点击的哪个按钮，按钮见functional.py
+ """
+ print(llm_kwargs["llm_model"])
+
+ if additional_fn is not None:
+ import core_functional
+ importlib.reload(core_functional) # 热更新prompt
+ core_functional = core_functional.get_core_functions()
+ if "PreProcess" in core_functional[additional_fn]: inputs = core_functional[additional_fn]["PreProcess"](inputs) # 获取预处理函数（如果有的话）
+ inputs = core_functional[additional_fn]["Prefix"] + inputs + core_functional[additional_fn]["Suffix"]
+
+ raw_input = inputs
+ logging.info(f'[raw_input] {raw_input}')
+ chatbot.append((inputs, ""))
+ yield from update_ui(chatbot=chatbot, history=history, msg="等待响应") # 刷新界面
+
+
+ payload = generate_azure_payload(inputs, llm_kwargs, history, system_prompt, stream)
+
+ history.append(inputs); history.append("")
+
+ retry = 0
+ while True:
+ try:
+
+ openai.api_type = "azure"
+ openai.api_version = AZURE_API_VERSION
+ openai.api_base = AZURE_ENDPOINT
+ openai.api_key = AZURE_API_KEY
+ response = openai.ChatCompletion.create(timeout=TIMEOUT_SECONDS, **payload);break
+
+ except:
+ retry += 1
+ chatbot[-1] = ((chatbot[-1][0], "获取response失败，重试中。。。"))
+ retry_msg = f"，正在重试 ({retry}/{MAX_RETRY}) ……" if MAX_RETRY > 0 else ""
+ yield from update_ui(chatbot=chatbot, history=history, msg="请求超时"+retry_msg) # 刷新界面
+ if retry > MAX_RETRY: raise TimeoutError
+
+ gpt_replying_buffer = ""
+ is_head_of_the_stream = True
+ if stream:
+
+ stream_response = response
+
+ while True:
+ try:
+ chunk = next(stream_response)
+
+ except StopIteration:
+ from toolbox import regular_txt_to_markdown; tb_str = '```\n' + trimmed_format_exc() + '```'
+ chatbot[-1] = (chatbot[-1][0], f"[Local Message] 远程返回错误: \n\n{tb_str} \n\n{regular_txt_to_markdown(chunk)}")
+ yield from update_ui(chatbot=chatbot, history=history, msg="远程返回错误:" + chunk) # 刷新界面
+ return
+
+ if is_head_of_the_stream and (r'"object":"error"' not in chunk):
+ # 数据流的第一帧不携带content
+ is_head_of_the_stream = False; continue
+
+ if chunk:
+ #print(chunk)
+ try:
+ if "delta" in chunk["choices"][0]:
+ if chunk["choices"][0]["finish_reason"] == "stop":
+ logging.info(f'[response] {gpt_replying_buffer}')
+ break
+ status_text = f"finish_reason: {chunk['choices'][0]['finish_reason']}"
+ gpt_replying_buffer = gpt_replying_buffer + chunk["choices"][0]["delta"]["content"]
+
+ history[-1] = gpt_replying_buffer
+ chatbot[-1] = (history[-2], history[-1])
+ yield from update_ui(chatbot=chatbot, history=history, msg=status_text) # 刷新界面
+
+ except Exception as e:
+ traceback.print_exc()
+ yield from update_ui(chatbot=chatbot, history=history, msg="Json解析不合常规") # 刷新界面
+ chunk = get_full_error(chunk, stream_response)
+
+ error_msg = chunk
+ yield from update_ui(chatbot=chatbot, history=history, msg="Json异常" + error_msg) # 刷新界面
+ return
+
+
+def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="", observe_window=None, console_slience=False):
+ """
+ 发送至AZURE OPENAI API，等待回复，一次性完成，不显示中间过程。但内部用stream的方法避免中途网线被掐。
+ inputs：
+ 是本次问询的输入
+ sys_prompt:
+ 系统静默prompt
+ llm_kwargs：
+ chatGPT的内部调优参数
+ history：
+ 是之前的对话列表
+ observe_window = None：
+ 用于负责跨越线程传递已经输出的部分，大部分时候仅仅为了fancy的视觉效果，留空即可。observe_window[0]：观测窗。observe_window[1]：看门狗
+ """
+ watch_dog_patience = 5 # 看门狗的耐心, 设置5秒即可
+ payload = generate_azure_payload(inputs, llm_kwargs, history, system_prompt=sys_prompt, stream=True)
+ retry = 0
+ while True:
+
+ try:
+ openai.api_type = "azure"
+ openai.api_version = AZURE_API_VERSION
+ openai.api_base = AZURE_ENDPOINT
+ openai.api_key = AZURE_API_KEY
+ response = openai.ChatCompletion.create(timeout=TIMEOUT_SECONDS, **payload);break
+
+ except:
+ retry += 1
+ traceback.print_exc()
+ if retry > MAX_RETRY: raise TimeoutError
+ if MAX_RETRY!=0: print(f'请求超时，正在重试 ({retry}/{MAX_RETRY}) ……')
+
+
+ stream_response = response
+ result = ''
+ while True:
+ try: chunk = next(stream_response)
+ except StopIteration:
+ break
+ except:
+ chunk = next(stream_response) # 失败了，重试一次？再失败就没办法了。
+
+ if len(chunk)==0: continue
+ if not chunk.startswith('data:'):
+ error_msg = get_full_error(chunk, stream_response)
+ if "reduce the length" in error_msg:
+ raise ConnectionAbortedError("AZURE OPENAI API拒绝了请求:" + error_msg)
+ else:
+ raise RuntimeError("AZURE OPENAI API拒绝了请求：" + error_msg)
+ if ('data: [DONE]' in chunk): break
+
+ delta = chunk["delta"]
+ if len(delta) == 0: break
+ if "role" in delta: continue
+ if "content" in delta:
+ result += delta["content"]
+ if not console_slience: print(delta["content"], end='')
+ if observe_window is not None:
+ # 观测窗，把已经获取的数据显示出去
+ if len(observe_window) >= 1: observe_window[0] += delta["content"]
+ # 看门狗，如果超过期限没有喂狗，则终止
+ if len(observe_window) >= 2:
+ if (time.time()-observe_window[1]) > watch_dog_patience:
+ raise RuntimeError("用户取消了程序。")
+ else: raise RuntimeError("意外Json结构："+delta)
+ if chunk['finish_reason'] == 'length':
+ raise ConnectionAbortedError("正常结束，但显示Token不足，导致输出不完整，请削减单次输入的文本量。")
+ return result
+
+
+def generate_azure_payload(inputs, llm_kwargs, history, system_prompt, stream):
+ """
+ 整合所有信息，选择LLM模型，生成 azure openai api请求，为发送请求做准备
+ """
+
+ conversation_cnt = len(history) // 2
+
+ messages = [{"role": "system", "content": system_prompt}]
+ if conversation_cnt:
+ for index in range(0, 2*conversation_cnt, 2):
+ what_i_have_asked = {}
+ what_i_have_asked["role"] = "user"
+ what_i_have_asked["content"] = history[index]
+ what_gpt_answer = {}
+ what_gpt_answer["role"] = "assistant"
+ what_gpt_answer["content"] = history[index+1]
+ if what_i_have_asked["content"] != "":
+ if what_gpt_answer["content"] == "": continue
+ messages.append(what_i_have_asked)
+ messages.append(what_gpt_answer)
+ else:
+ messages[-1]['content'] = what_gpt_answer['content']
+
+ what_i_ask_now = {}
+ what_i_ask_now["role"] = "user"
+ what_i_ask_now["content"] = inputs
+ messages.append(what_i_ask_now)
+
+ payload = {
+ "model": llm_kwargs['llm_model'],
+ "messages": messages,
+ "temperature": llm_kwargs['temperature'], # 1.0,
+ "top_p": llm_kwargs['top_p'], # 1.0,
+ "n": 1,
+ "stream": stream,
+ "presence_penalty": 0,
+ "frequency_penalty": 0,
+ "engine": AZURE_ENGINE
+ }
+ try:
+ print(f" {llm_kwargs['llm_model']} : {conversation_cnt} : {inputs[:100]} ..........")
+ except:
+ print('输入中可能存在乱码。')
+ return payload
+
+
diff --git a/request_llm/bridge_chatglm.py b/request_llm/bridge_chatglm.py
new file mode 100644
index 0000000000000000000000000000000000000000..6dac86395da134aa896da9d9a7c84ccd94e795d0
--- /dev/null
+++ b/request_llm/bridge_chatglm.py
@@ -0,0 +1,166 @@
+
+from transformers import AutoModel, AutoTokenizer
+import time
+import threading
+import importlib
+from toolbox import update_ui, get_conf
+from multiprocessing import Process, Pipe
+
+load_message = "ChatGLM尚未加载，加载需要一段时间。注意，取决于`config.py`的配置，ChatGLM消耗大量的内存（CPU）或显存（GPU），也许会导致低配计算机卡死 ……"
+
+#################################################################################
+class GetGLMHandle(Process):
+ def __init__(self):
+ super().__init__(daemon=True)
+ self.parent, self.child = Pipe()
+ self.chatglm_model = None
+ self.chatglm_tokenizer = None
+ self.info = ""
+ self.success = True
+ self.check_dependency()
+ self.start()
+ self.threadLock = threading.Lock()
+
+ def check_dependency(self):
+ try:
+ import sentencepiece
+ self.info = "依赖检测通过"
+ self.success = True
+ except:
+ self.info = "缺少ChatGLM的依赖，如果要使用ChatGLM，除了基础的pip依赖以外，您还需要运行`pip install -r request_llm/requirements_chatglm.txt`安装ChatGLM的依赖。"
+ self.success = False
+
+ def ready(self):
+ return self.chatglm_model is not None
+
+ def run(self):
+ # 子进程执行
+ # 第一次运行，加载参数
+ retry = 0
+ LOCAL_MODEL_QUANT, device = get_conf('LOCAL_MODEL_QUANT', 'LOCAL_MODEL_DEVICE')
+
+ if LOCAL_MODEL_QUANT == "INT4": # INT4
+ _model_name_ = "THUDM/chatglm2-6b-int4"
+ elif LOCAL_MODEL_QUANT == "INT8": # INT8
+ _model_name_ = "THUDM/chatglm2-6b-int8"
+ else:
+ _model_name_ = "THUDM/chatglm2-6b" # FP16
+
+ while True:
+ try:
+ if self.chatglm_model is None:
+ self.chatglm_tokenizer = AutoTokenizer.from_pretrained(_model_name_, trust_remote_code=True)
+ if device=='cpu':
+ self.chatglm_model = AutoModel.from_pretrained(_model_name_, trust_remote_code=True).float()
+ else:
+ self.chatglm_model = AutoModel.from_pretrained(_model_name_, trust_remote_code=True).half().cuda()
+ self.chatglm_model = self.chatglm_model.eval()
+ break
+ else:
+ break
+ except:
+ retry += 1
+ if retry > 3:
+ self.child.send('[Local Message] Call ChatGLM fail 不能正常加载ChatGLM的参数。')
+ raise RuntimeError("不能正常加载ChatGLM的参数！")
+
+ while True:
+ # 进入任务等待状态
+ kwargs = self.child.recv()
+ # 收到消息，开始请求
+ try:
+ for response, history in self.chatglm_model.stream_chat(self.chatglm_tokenizer, **kwargs):
+ self.child.send(response)
+ # # 中途接收可能的终止指令（如果有的话）
+ # if self.child.poll():
+ # command = self.child.recv()
+ # if command == '[Terminate]': break
+ except:
+ from toolbox import trimmed_format_exc
+ self.child.send('[Local Message] Call ChatGLM fail.' + '\n```\n' + trimmed_format_exc() + '\n```\n')
+ # 请求处理结束，开始下一个循环
+ self.child.send('[Finish]')
+
+ def stream_chat(self, **kwargs):
+ # 主进程执行
+ self.threadLock.acquire()
+ self.parent.send(kwargs)
+ while True:
+ res = self.parent.recv()
+ if res != '[Finish]':
+ yield res
+ else:
+ break
+ self.threadLock.release()
+
+global glm_handle
+glm_handle = None
+#################################################################################
+def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="", observe_window=[], console_slience=False):
+ """
+ 多线程方法
+ 函数的说明请见 request_llm/bridge_all.py
+ """
+ global glm_handle
+ if glm_handle is None:
+ glm_handle = GetGLMHandle()
+ if len(observe_window) >= 1: observe_window[0] = load_message + "\n\n" + glm_handle.info
+ if not glm_handle.success:
+ error = glm_handle.info
+ glm_handle = None
+ raise RuntimeError(error)
+
+ # chatglm 没有 sys_prompt 接口，因此把prompt加入 history
+ history_feedin = []
+ history_feedin.append(["What can I do?", sys_prompt])
+ for i in range(len(history)//2):
+ history_feedin.append([history[2*i], history[2*i+1]] )
+
+ watch_dog_patience = 5 # 看门狗 (watchdog) 的耐心, 设置5秒即可
+ response = ""
+ for response in glm_handle.stream_chat(query=inputs, history=history_feedin, max_length=llm_kwargs['max_length'], top_p=llm_kwargs['top_p'], temperature=llm_kwargs['temperature']):
+ if len(observe_window) >= 1: observe_window[0] = response
+ if len(observe_window) >= 2:
+ if (time.time()-observe_window[1]) > watch_dog_patience:
+ raise RuntimeError("程序终止。")
+ return response
+
+
+
+def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_prompt='', stream = True, additional_fn=None):
+ """
+ 单线程方法
+ 函数的说明请见 request_llm/bridge_all.py
+ """
+ chatbot.append((inputs, ""))
+
+ global glm_handle
+ if glm_handle is None:
+ glm_handle = GetGLMHandle()
+ chatbot[-1] = (inputs, load_message + "\n\n" + glm_handle.info)
+ yield from update_ui(chatbot=chatbot, history=[])
+ if not glm_handle.success:
+ glm_handle = None
+ return
+
+ if additional_fn is not None:
+ from core_functional import handle_core_functionality
+ inputs, history = handle_core_functionality(additional_fn, inputs, history, chatbot)
+
+ # 处理历史信息
+ history_feedin = []
+ history_feedin.append(["What can I do?", system_prompt] )
+ for i in range(len(history)//2):
+ history_feedin.append([history[2*i], history[2*i+1]] )
+
+ # 开始接收chatglm的回复
+ response = "[Local Message]: 等待ChatGLM响应中 ..."
+ for response in glm_handle.stream_chat(query=inputs, history=history_feedin, max_length=llm_kwargs['max_length'], top_p=llm_kwargs['top_p'], temperature=llm_kwargs['temperature']):
+ chatbot[-1] = (inputs, response)
+ yield from update_ui(chatbot=chatbot, history=history)
+
+ # 总结输出
+ if response == "[Local Message]: 等待ChatGLM响应中 ...":
+ response = "[Local Message]: ChatGLM响应异常 ..."
+ history.extend([inputs, response])
+ yield from update_ui(chatbot=chatbot, history=history)
diff --git a/request_llms/bridge_chatglmft.py b/request_llm/bridge_chatglmft.py
similarity index 93%
rename from request_llms/bridge_chatglmft.py
rename to request_llm/bridge_chatglmft.py
index d812bae3c36dc22e6c40e78b54e0fbbda665e989..71af94213e1824f8c664d5759812b6057d3b29a2 100644
--- a/request_llms/bridge_chatglmft.py
+++ b/request_llm/bridge_chatglmft.py
@@ -44,7 +44,7 @@ class GetGLMFTHandle(Process):
self.info = "依赖检测通过"
self.success = True
except:
- self.info = "缺少ChatGLMFT的依赖，如果要使用ChatGLMFT，除了基础的pip依赖以外，您还需要运行`pip install -r request_llms/requirements_chatglm.txt`安装ChatGLM的依赖。"
+ self.info = "缺少ChatGLMFT的依赖，如果要使用ChatGLMFT，除了基础的pip依赖以外，您还需要运行`pip install -r request_llm/requirements_chatglm.txt`安装ChatGLM的依赖。"
self.success = False
def ready(self):
@@ -59,11 +59,11 @@ class GetGLMFTHandle(Process):
if self.chatglmft_model is None:
from transformers import AutoConfig
import torch
- # conf = 'request_llms/current_ptune_model.json'
+ # conf = 'request_llm/current_ptune_model.json'
# if not os.path.exists(conf): raise RuntimeError('找不到微调模型信息')
# with open(conf, 'r', encoding='utf8') as f:
# model_args = json.loads(f.read())
- CHATGLM_PTUNING_CHECKPOINT = get_conf('CHATGLM_PTUNING_CHECKPOINT')
+ CHATGLM_PTUNING_CHECKPOINT, = get_conf('CHATGLM_PTUNING_CHECKPOINT')
assert os.path.exists(CHATGLM_PTUNING_CHECKPOINT), "找不到微调模型检查点"
conf = os.path.join(CHATGLM_PTUNING_CHECKPOINT, "config.json")
with open(conf, 'r', encoding='utf8') as f:
@@ -87,7 +87,7 @@ class GetGLMFTHandle(Process):
new_prefix_state_dict[k[len("transformer.prefix_encoder."):]] = v
model.transformer.prefix_encoder.load_state_dict(new_prefix_state_dict)
- if model_args['quantization_bit'] is not None and model_args['quantization_bit'] != 0:
+ if model_args['quantization_bit'] is not None:
print(f"Quantized to {model_args['quantization_bit']} bit")
model = model.quantize(model_args['quantization_bit'])
model = model.cuda()
@@ -140,7 +140,7 @@ glmft_handle = None
def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="", observe_window=[], console_slience=False):
"""
多线程方法
- 函数的说明请见 request_llms/bridge_all.py
+ 函数的说明请见 request_llm/bridge_all.py
"""
global glmft_handle
if glmft_handle is None:
@@ -171,7 +171,7 @@ def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="",
def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_prompt='', stream = True, additional_fn=None):
"""
单线程方法
- 函数的说明请见 request_llms/bridge_all.py
+ 函数的说明请见 request_llm/bridge_all.py
"""
chatbot.append((inputs, ""))
@@ -195,13 +195,13 @@ def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_promp
history_feedin.append([history[2*i], history[2*i+1]] )
# 开始接收chatglmft的回复
- response = "[Local Message] 等待ChatGLMFT响应中 ..."
+ response = "[Local Message]: 等待ChatGLMFT响应中 ..."
for response in glmft_handle.stream_chat(query=inputs, history=history_feedin, max_length=llm_kwargs['max_length'], top_p=llm_kwargs['top_p'], temperature=llm_kwargs['temperature']):
chatbot[-1] = (inputs, response)
yield from update_ui(chatbot=chatbot, history=history)
# 总结输出
- if response == "[Local Message] 等待ChatGLMFT响应中 ...":
- response = "[Local Message] ChatGLMFT响应异常 ..."
+ if response == "[Local Message]: 等待ChatGLMFT响应中 ...":
+ response = "[Local Message]: ChatGLMFT响应异常 ..."
history.extend([inputs, response])
yield from update_ui(chatbot=chatbot, history=history)
diff --git a/request_llms/bridge_chatglmonnx.py b/request_llm/bridge_chatglmonnx.py
similarity index 82%
rename from request_llms/bridge_chatglmonnx.py
rename to request_llm/bridge_chatglmonnx.py
index 4b905718f63089c1355d244d61c67df07c3dc521..594bcca15f04c7d9790da95fee2a1d51252c07d1 100644
--- a/request_llms/bridge_chatglmonnx.py
+++ b/request_llm/bridge_chatglmonnx.py
@@ -1,5 +1,5 @@
model_name = "ChatGLM-ONNX"
-cmd_to_install = "`pip install -r request_llms/requirements_chatglm_onnx.txt`"
+cmd_to_install = "`pip install -r request_llm/requirements_chatglm_onnx.txt`"
from transformers import AutoModel, AutoTokenizer
@@ -8,7 +8,7 @@ import threading
import importlib
from toolbox import update_ui, get_conf
from multiprocessing import Process, Pipe
-from .local_llm_class import LocalLLMHandle, get_local_llm_predict_fns
+from .local_llm_class import LocalLLMHandle, get_local_llm_predict_fns, SingletonLocalLLM
from .chatglmoonx import ChatGLMModel, chat_template
@@ -17,6 +17,7 @@ from .chatglmoonx import ChatGLMModel, chat_template
# ------------------------------------------------------------------------------------------------------------------------
# 🔌💻 Local Model
# ------------------------------------------------------------------------------------------------------------------------
+@SingletonLocalLLM
class GetONNXGLMHandle(LocalLLMHandle):
def load_model_info(self):
@@ -27,13 +28,13 @@ class GetONNXGLMHandle(LocalLLMHandle):
def load_model_and_tokenizer(self):
# 🏃♂️🏃♂️🏃♂️ 子进程执行
import os, glob
- if not len(glob.glob("./request_llms/ChatGLM-6b-onnx-u8s8/chatglm-6b-int8-onnx-merged/*.bin")) >= 7: # 该模型有七个 bin 文件
+ if not len(glob.glob("./request_llm/ChatGLM-6b-onnx-u8s8/chatglm-6b-int8-onnx-merged/*.bin")) >= 7: # 该模型有七个 bin 文件
from huggingface_hub import snapshot_download
- snapshot_download(repo_id="K024/ChatGLM-6b-onnx-u8s8", local_dir="./request_llms/ChatGLM-6b-onnx-u8s8")
+ snapshot_download(repo_id="K024/ChatGLM-6b-onnx-u8s8", local_dir="./request_llm/ChatGLM-6b-onnx-u8s8")
def create_model():
return ChatGLMModel(
- tokenizer_path = "./request_llms/ChatGLM-6b-onnx-u8s8/chatglm-6b-int8-onnx-merged/sentencepiece.model",
- onnx_model_path = "./request_llms/ChatGLM-6b-onnx-u8s8/chatglm-6b-int8-onnx-merged/chatglm-6b-int8.onnx"
+ tokenizer_path = "./request_llm/ChatGLM-6b-onnx-u8s8/chatglm-6b-int8-onnx-merged/sentencepiece.model",
+ onnx_model_path = "./request_llm/ChatGLM-6b-onnx-u8s8/chatglm-6b-int8-onnx-merged/chatglm-6b-int8.onnx"
)
self._model = create_model()
return self._model, None
diff --git a/request_llms/bridge_chatgpt.py b/request_llm/bridge_chatgpt.py
similarity index 74%
rename from request_llms/bridge_chatgpt.py
rename to request_llm/bridge_chatgpt.py
index ecb8423b4621dfe4ccedf3c679e8b007389112ab..a1b6ba47d3e91790ad50e8de9a583267ae4dd3ec 100644
--- a/request_llms/bridge_chatgpt.py
+++ b/request_llm/bridge_chatgpt.py
@@ -7,7 +7,8 @@
1. predict: 正常对话时使用，具备完备的交互功能，不可多线程
具备多线程调用能力的函数
- 2. predict_no_ui_long_connection：支持多线程
+ 2. predict_no_ui：高级实验性功能模块调用，不会实时显示在界面上，参数简单，可以多线程并行，方便实现复杂的功能逻辑
+ 3. predict_no_ui_long_connection：在实验过程中发现调用predict_no_ui处理长文档时，和openai的连接容易断掉，这个函数用stream的方式解决这个问题，同样支持多线程
"""
import json
@@ -17,13 +18,12 @@ import logging
import traceback
import requests
import importlib
-import random
# config_private.py放自己的秘密如API和代理网址
# 读取时首先看是否存在私密的config_private配置文件（不受git管控），如果有，则覆盖原config文件
from toolbox import get_conf, update_ui, is_any_api_key, select_api_key, what_keys, clip_history, trimmed_format_exc, is_the_upload_folder
-proxies, TIMEOUT_SECONDS, MAX_RETRY, API_ORG, AZURE_CFG_ARRAY = \
- get_conf('proxies', 'TIMEOUT_SECONDS', 'MAX_RETRY', 'API_ORG', 'AZURE_CFG_ARRAY')
+proxies, TIMEOUT_SECONDS, MAX_RETRY, API_ORG = \
+ get_conf('proxies', 'TIMEOUT_SECONDS', 'MAX_RETRY', 'API_ORG')
timeout_bot_msg = '[Local Message] Request timeout. Network error. Please check proxy settings in config.py.' + \
'网络错误，检查代理服务器是否可用，以及代理设置的格式是否正确，格式须是[协议]://[地址]:[端口]，缺一不可。'
@@ -39,34 +39,6 @@ def get_full_error(chunk, stream_response):
break
return chunk
-def decode_chunk(chunk):
- # 提前读取一些信息 （用于判断异常）
- chunk_decoded = chunk.decode()
- chunkjson = None
- has_choices = False
- choice_valid = False
- has_content = False
- has_role = False
- try:
- chunkjson = json.loads(chunk_decoded[6:])
- has_choices = 'choices' in chunkjson
- if has_choices: choice_valid = (len(chunkjson['choices']) > 0)
- if has_choices and choice_valid: has_content = ("content" in chunkjson['choices'][0]["delta"])
- if has_content: has_content = (chunkjson['choices'][0]["delta"]["content"] is not None)
- if has_choices and choice_valid: has_role = "role" in chunkjson['choices'][0]["delta"]
- except:
- pass
- return chunk_decoded, chunkjson, has_choices, choice_valid, has_content, has_role
-
-from functools import lru_cache
-@lru_cache(maxsize=32)
-def verify_endpoint(endpoint):
- """
- 检查endpoint是否可用
- """
- if "你亲手写的api名称" in endpoint:
- raise ValueError("Endpoint不正确, 请检查AZURE_ENDPOINT的配置! 当前的Endpoint为:" + endpoint)
- return endpoint
def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="", observe_window=None, console_slience=False):
"""
@@ -89,7 +61,7 @@ def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="",
try:
# make a POST request to the API endpoint, stream=False
from .bridge_all import model_info
- endpoint = verify_endpoint(model_info[llm_kwargs['llm_model']]['endpoint'])
+ endpoint = model_info[llm_kwargs['llm_model']]['endpoint']
response = requests.post(endpoint, headers=headers, proxies=proxies,
json=payload, stream=True, timeout=TIMEOUT_SECONDS); break
except requests.exceptions.ReadTimeout as e:
@@ -98,31 +70,24 @@ def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="",
if retry > MAX_RETRY: raise TimeoutError
if MAX_RETRY!=0: print(f'请求超时，正在重试 ({retry}/{MAX_RETRY}) ……')
- stream_response = response.iter_lines()
+ stream_response = response.iter_lines()
result = ''
json_data = None
while True:
- try: chunk = next(stream_response)
+ try: chunk = next(stream_response).decode()
except StopIteration:
break
except requests.exceptions.ConnectionError:
- chunk = next(stream_response) # 失败了，重试一次？再失败就没办法了。
- chunk_decoded, chunkjson, has_choices, choice_valid, has_content, has_role = decode_chunk(chunk)
- if len(chunk_decoded)==0: continue
- if not chunk_decoded.startswith('data:'):
- error_msg = get_full_error(chunk, stream_response).decode()
+ chunk = next(stream_response).decode() # 失败了，重试一次？再失败就没办法了。
+ if len(chunk)==0: continue
+ if not chunk.startswith('data:'):
+ error_msg = get_full_error(chunk.encode('utf8'), stream_response).decode()
if "reduce the length" in error_msg:
raise ConnectionAbortedError("OpenAI拒绝了请求:" + error_msg)
- elif """type":"upstream_error","param":"307""" in error_msg:
- raise ConnectionAbortedError("正常结束，但显示Token不足，导致输出不完整，请削减单次输入的文本量。")
else:
raise RuntimeError("OpenAI拒绝了请求：" + error_msg)
- if ('data: [DONE]' in chunk_decoded): break # api2d 正常完成
- # 提前读取一些信息 （用于判断异常）
- if has_choices and not choice_valid:
- # 一些垃圾第三方接口的出现这样的错误
- continue
- json_data = chunkjson['choices'][0]
+ if ('data: [DONE]' in chunk): break # api2d 正常完成
+ json_data = json.loads(chunk.lstrip('data:'))['choices'][0]
delta = json_data["delta"]
if len(delta) == 0: break
if "role" in delta: continue
@@ -188,22 +153,14 @@ def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_promp
yield from update_ui(chatbot=chatbot, history=history, msg="api-key不满足要求") # 刷新界面
return
- # 检查endpoint是否合法
- try:
- from .bridge_all import model_info
- endpoint = verify_endpoint(model_info[llm_kwargs['llm_model']]['endpoint'])
- except:
- tb_str = '```\n' + trimmed_format_exc() + '```'
- chatbot[-1] = (inputs, tb_str)
- yield from update_ui(chatbot=chatbot, history=history, msg="Endpoint不满足要求") # 刷新界面
- return
-
history.append(inputs); history.append("")
retry = 0
while True:
try:
# make a POST request to the API endpoint, stream=True
+ from .bridge_all import model_info
+ endpoint = model_info[llm_kwargs['llm_model']]['endpoint']
response = requests.post(endpoint, headers=headers, proxies=proxies,
json=payload, stream=True, timeout=TIMEOUT_SECONDS);break
except:
@@ -234,39 +191,23 @@ def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_promp
yield from update_ui(chatbot=chatbot, history=history, msg="非OpenAI官方接口返回了错误:" + chunk.decode()) # 刷新界面
return
- # 提前读取一些信息 （用于判断异常）
- chunk_decoded, chunkjson, has_choices, choice_valid, has_content, has_role = decode_chunk(chunk)
-
+ chunk_decoded = chunk.decode()
if is_head_of_the_stream and (r'"object":"error"' not in chunk_decoded) and (r"content" not in chunk_decoded):
# 数据流的第一帧不携带content
is_head_of_the_stream = False; continue
if chunk:
try:
- if has_choices and not choice_valid:
- # 一些垃圾第三方接口的出现这样的错误
- continue
- if ('data: [DONE]' not in chunk_decoded) and len(chunk_decoded) > 0 and (chunkjson is None):
- # 传递进来一些奇怪的东西
- raise ValueError(f'无法读取以下数据，请检查配置。\n\n{chunk_decoded}')
# 前者是API2D的结束条件，后者是OPENAI的结束条件
- if ('data: [DONE]' in chunk_decoded) or (len(chunkjson['choices'][0]["delta"]) == 0):
+ if ('data: [DONE]' in chunk_decoded) or (len(json.loads(chunk_decoded[6:])['choices'][0]["delta"]) == 0):
# 判定为数据流的结束，gpt_replying_buffer也写完了
logging.info(f'[response] {gpt_replying_buffer}')
break
# 处理数据流的主体
+ chunkjson = json.loads(chunk_decoded[6:])
status_text = f"finish_reason: {chunkjson['choices'][0].get('finish_reason', 'null')}"
# 如果这里抛出异常，一般是文本过长，详情见get_full_error的输出
- if has_content:
- # 正常情况
- gpt_replying_buffer = gpt_replying_buffer + chunkjson['choices'][0]["delta"]["content"]
- elif has_role:
- # 一些第三方接口的出现这样的错误，兼容一下吧
- continue
- else:
- # 一些垃圾第三方接口的出现这样的错误
- gpt_replying_buffer = gpt_replying_buffer + chunkjson['choices'][0]["delta"]["content"]
-
+ gpt_replying_buffer = gpt_replying_buffer + chunkjson['choices'][0]["delta"]["content"]
history[-1] = gpt_replying_buffer
chatbot[-1] = (history[-2], history[-1])
yield from update_ui(chatbot=chatbot, history=history, msg=status_text) # 刷新界面
@@ -298,8 +239,6 @@ def handle_error(inputs, llm_kwargs, chatbot, history, chunk_decoded, error_msg)
chatbot[-1] = (chatbot[-1][0], "[Local Message] Your account is not active. OpenAI以账户失效为由, 拒绝服务." + openai_website)
elif "associated with a deactivated account" in error_msg:
chatbot[-1] = (chatbot[-1][0], "[Local Message] You are associated with a deactivated account. OpenAI以账户失效为由, 拒绝服务." + openai_website)
- elif "API key has been deactivated" in error_msg:
- chatbot[-1] = (chatbot[-1][0], "[Local Message] API key has been deactivated. OpenAI以账户失效为由, 拒绝服务." + openai_website)
elif "bad forward key" in error_msg:
chatbot[-1] = (chatbot[-1][0], "[Local Message] Bad forward key. API2D账户额度不足.")
elif "Not enough point" in error_msg:
@@ -324,11 +263,7 @@ def generate_payload(inputs, llm_kwargs, history, system_prompt, stream):
"Authorization": f"Bearer {api_key}"
}
if API_ORG.startswith('org-'): headers.update({"OpenAI-Organization": API_ORG})
- if llm_kwargs['llm_model'].startswith('azure-'):
- headers.update({"api-key": api_key})
- if llm_kwargs['llm_model'] in AZURE_CFG_ARRAY.keys():
- azure_api_key_unshared = AZURE_CFG_ARRAY[llm_kwargs['llm_model']]["AZURE_API_KEY"]
- headers.update({"api-key": azure_api_key_unshared})
+ if llm_kwargs['llm_model'].startswith('azure-'): headers.update({"api-key": api_key})
conversation_cnt = len(history) // 2
@@ -353,23 +288,9 @@ def generate_payload(inputs, llm_kwargs, history, system_prompt, stream):
what_i_ask_now["role"] = "user"
what_i_ask_now["content"] = inputs
messages.append(what_i_ask_now)
- model = llm_kwargs['llm_model']
- if llm_kwargs['llm_model'].startswith('api2d-'):
- model = llm_kwargs['llm_model'][len('api2d-'):]
-
- if model == "gpt-3.5-random": # 随机选择, 绕过openai访问频率限制
- model = random.choice([
- "gpt-3.5-turbo",
- "gpt-3.5-turbo-16k",
- "gpt-3.5-turbo-1106",
- "gpt-3.5-turbo-0613",
- "gpt-3.5-turbo-16k-0613",
- "gpt-3.5-turbo-0301",
- ])
- logging.info("Random select model:" + model)
payload = {
- "model": model,
+ "model": llm_kwargs['llm_model'].strip('api2d-'),
"messages": messages,
"temperature": llm_kwargs['temperature'], # 1.0,
"top_p": llm_kwargs['top_p'], # 1.0,
diff --git a/request_llms/bridge_chatgpt_website.py b/request_llm/bridge_chatgpt_website.py
similarity index 97%
rename from request_llms/bridge_chatgpt_website.py
rename to request_llm/bridge_chatgpt_website.py
index f2f0709099961884f0e8f7644d3cfc9a72d11333..7f3147b1d6dfb0c3889b2a21bf9ebcea0b17ca5f 100644
--- a/request_llms/bridge_chatgpt_website.py
+++ b/request_llm/bridge_chatgpt_website.py
@@ -7,7 +7,8 @@
1. predict: 正常对话时使用，具备完备的交互功能，不可多线程
具备多线程调用能力的函数
- 2. predict_no_ui_long_connection：支持多线程
+ 2. predict_no_ui：高级实验性功能模块调用，不会实时显示在界面上，参数简单，可以多线程并行，方便实现复杂的功能逻辑
+ 3. predict_no_ui_long_connection：在实验过程中发现调用predict_no_ui处理长文档时，和openai的连接容易断掉，这个函数用stream的方式解决这个问题，同样支持多线程
"""
import json
diff --git a/request_llms/bridge_claude.py b/request_llm/bridge_claude.py
similarity index 97%
rename from request_llms/bridge_claude.py
rename to request_llm/bridge_claude.py
index 42b75052f80977a29efec1c3755bcbf56ff47d76..6084b1f15c9832fd11a36bb58d8187f4e2a7a931 100644
--- a/request_llms/bridge_claude.py
+++ b/request_llm/bridge_claude.py
@@ -7,7 +7,7 @@
1. predict: 正常对话时使用，具备完备的交互功能，不可多线程
具备多线程调用能力的函数
- 2. predict_no_ui_long_connection：支持多线程
+ 2. predict_no_ui_long_connection：在实验过程中发现调用predict_no_ui处理长文档时，和openai的连接容易断掉，这个函数用stream的方式解决这个问题，同样支持多线程
"""
import os
diff --git a/request_llms/bridge_internlm.py b/request_llm/bridge_internlm.py
similarity index 91%
rename from request_llms/bridge_internlm.py
rename to request_llm/bridge_internlm.py
index b2be36a4e692b6d6c0513174d13f35c9d055812c..0ec65b641d366b572640f9d9690e1d9ab86ed40b 100644
--- a/request_llms/bridge_internlm.py
+++ b/request_llm/bridge_internlm.py
@@ -1,13 +1,13 @@
model_name = "InternLM"
-cmd_to_install = "`pip install -r request_llms/requirements_chatglm.txt`"
+cmd_to_install = "`pip install -r request_llm/requirements_chatglm.txt`"
from transformers import AutoModel, AutoTokenizer
import time
import threading
import importlib
-from toolbox import update_ui, get_conf, ProxyNetworkActivate
+from toolbox import update_ui, get_conf
from multiprocessing import Process, Pipe
-from .local_llm_class import LocalLLMHandle, get_local_llm_predict_fns
+from .local_llm_class import LocalLLMHandle, get_local_llm_predict_fns, SingletonLocalLLM
# ------------------------------------------------------------------------------------------------------------------------
@@ -34,6 +34,7 @@ def combine_history(prompt, hist):
# ------------------------------------------------------------------------------------------------------------------------
# 🔌💻 Local Model
# ------------------------------------------------------------------------------------------------------------------------
+@SingletonLocalLLM
class GetInternlmHandle(LocalLLMHandle):
def load_model_info(self):
@@ -51,16 +52,15 @@ class GetInternlmHandle(LocalLLMHandle):
# 🏃♂️🏃♂️🏃♂️ 子进程执行
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
- device = get_conf('LOCAL_MODEL_DEVICE')
- with ProxyNetworkActivate('Download_LLM'):
- if self._model is None:
- tokenizer = AutoTokenizer.from_pretrained("internlm/internlm-chat-7b", trust_remote_code=True)
- if device=='cpu':
- model = AutoModelForCausalLM.from_pretrained("internlm/internlm-chat-7b", trust_remote_code=True).to(torch.bfloat16)
- else:
- model = AutoModelForCausalLM.from_pretrained("internlm/internlm-chat-7b", trust_remote_code=True).to(torch.bfloat16).cuda()
-
- model = model.eval()
+ device, = get_conf('LOCAL_MODEL_DEVICE')
+ if self._model is None:
+ tokenizer = AutoTokenizer.from_pretrained("internlm/internlm-chat-7b", trust_remote_code=True)
+ if device=='cpu':
+ model = AutoModelForCausalLM.from_pretrained("internlm/internlm-chat-7b", trust_remote_code=True).to(torch.bfloat16)
+ else:
+ model = AutoModelForCausalLM.from_pretrained("internlm/internlm-chat-7b", trust_remote_code=True).to(torch.bfloat16).cuda()
+
+ model = model.eval()
return model, tokenizer
def llm_stream_generator(self, **kwargs):
@@ -94,9 +94,8 @@ class GetInternlmHandle(LocalLLMHandle):
inputs = tokenizer([prompt], padding=True, return_tensors="pt")
input_length = len(inputs["input_ids"][0])
- device = get_conf('LOCAL_MODEL_DEVICE')
for k, v in inputs.items():
- inputs[k] = v.to(device)
+ inputs[k] = v.cuda()
input_ids = inputs["input_ids"]
batch_size, input_ids_seq_length = input_ids.shape[0], input_ids.shape[-1]
if generation_config is None:
diff --git a/request_llms/bridge_jittorllms_llama.py b/request_llm/bridge_jittorllms_llama.py
similarity index 90%
rename from request_llms/bridge_jittorllms_llama.py
rename to request_llm/bridge_jittorllms_llama.py
index 2d3005e52d327839b08668a6413d462682baa046..d4853578fadff9f572c86f7f0fe79f9c8c8c1474 100644
--- a/request_llms/bridge_jittorllms_llama.py
+++ b/request_llm/bridge_jittorllms_llama.py
@@ -28,8 +28,8 @@ class GetGLMHandle(Process):
self.success = True
except:
from toolbox import trimmed_format_exc
- self.info = r"缺少jittorllms的依赖，如果要使用jittorllms，除了基础的pip依赖以外，您还需要运行`pip install -r request_llms/requirements_jittorllms.txt -i https://pypi.jittor.org/simple -I`"+\
- r"和`git clone https://gitlink.org.cn/jittor/JittorLLMs.git --depth 1 request_llms/jittorllms`两个指令来安装jittorllms的依赖（在项目根目录运行这两个指令）。" +\
+ self.info = r"缺少jittorllms的依赖，如果要使用jittorllms，除了基础的pip依赖以外，您还需要运行`pip install -r request_llm/requirements_jittorllms.txt -i https://pypi.jittor.org/simple -I`"+\
+ r"和`git clone https://gitlink.org.cn/jittor/JittorLLMs.git --depth 1 request_llm/jittorllms`两个指令来安装jittorllms的依赖（在项目根目录运行这两个指令）。" +\
r"警告：安装jittorllms依赖后将完全破坏现有的pytorch环境，建议使用docker环境！" + trimmed_format_exc()
self.success = False
@@ -45,15 +45,15 @@ class GetGLMHandle(Process):
env = os.environ.get("PATH", "")
os.environ["PATH"] = env.replace('/cuda/bin', '/x/bin')
root_dir_assume = os.path.abspath(os.path.dirname(__file__) + '/..')
- os.chdir(root_dir_assume + '/request_llms/jittorllms')
- sys.path.append(root_dir_assume + '/request_llms/jittorllms')
+ os.chdir(root_dir_assume + '/request_llm/jittorllms')
+ sys.path.append(root_dir_assume + '/request_llm/jittorllms')
validate_path() # validate path so you can run from base directory
def load_model():
import types
try:
if self.jittorllms_model is None:
- device = get_conf('LOCAL_MODEL_DEVICE')
+ device, = get_conf('LOCAL_MODEL_DEVICE')
from .jittorllms.models import get_model
# availabel_models = ["chatglm", "pangualpha", "llama", "chatrwkv"]
args_dict = {'model': 'llama'}
@@ -109,7 +109,7 @@ llama_glm_handle = None
def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="", observe_window=[], console_slience=False):
"""
多线程方法
- 函数的说明请见 request_llms/bridge_all.py
+ 函数的说明请见 request_llm/bridge_all.py
"""
global llama_glm_handle
if llama_glm_handle is None:
@@ -140,7 +140,7 @@ def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="",
def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_prompt='', stream = True, additional_fn=None):
"""
单线程方法
- 函数的说明请见 request_llms/bridge_all.py
+ 函数的说明请见 request_llm/bridge_all.py
"""
chatbot.append((inputs, ""))
@@ -163,13 +163,13 @@ def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_promp
history_feedin.append([history[2*i], history[2*i+1]] )
# 开始接收jittorllms的回复
- response = "[Local Message] 等待jittorllms响应中 ..."
+ response = "[Local Message]: 等待jittorllms响应中 ..."
for response in llama_glm_handle.stream_chat(query=inputs, history=history_feedin, system_prompt=system_prompt, max_length=llm_kwargs['max_length'], top_p=llm_kwargs['top_p'], temperature=llm_kwargs['temperature']):
chatbot[-1] = (inputs, response)
yield from update_ui(chatbot=chatbot, history=history)
# 总结输出
- if response == "[Local Message] 等待jittorllms响应中 ...":
- response = "[Local Message] jittorllms响应异常 ..."
+ if response == "[Local Message]: 等待jittorllms响应中 ...":
+ response = "[Local Message]: jittorllms响应异常 ..."
history.extend([inputs, response])
yield from update_ui(chatbot=chatbot, history=history)
diff --git a/request_llms/bridge_jittorllms_pangualpha.py b/request_llm/bridge_jittorllms_pangualpha.py
similarity index 90%
rename from request_llms/bridge_jittorllms_pangualpha.py
rename to request_llm/bridge_jittorllms_pangualpha.py
index 26401764c9bab23cc5412253128589b631c4c0b2..20a30213032e957113d6377d7c7f5a9912ea22b1 100644
--- a/request_llms/bridge_jittorllms_pangualpha.py
+++ b/request_llm/bridge_jittorllms_pangualpha.py
@@ -28,8 +28,8 @@ class GetGLMHandle(Process):
self.success = True
except:
from toolbox import trimmed_format_exc
- self.info = r"缺少jittorllms的依赖，如果要使用jittorllms，除了基础的pip依赖以外，您还需要运行`pip install -r request_llms/requirements_jittorllms.txt -i https://pypi.jittor.org/simple -I`"+\
- r"和`git clone https://gitlink.org.cn/jittor/JittorLLMs.git --depth 1 request_llms/jittorllms`两个指令来安装jittorllms的依赖（在项目根目录运行这两个指令）。" +\
+ self.info = r"缺少jittorllms的依赖，如果要使用jittorllms，除了基础的pip依赖以外，您还需要运行`pip install -r request_llm/requirements_jittorllms.txt -i https://pypi.jittor.org/simple -I`"+\
+ r"和`git clone https://gitlink.org.cn/jittor/JittorLLMs.git --depth 1 request_llm/jittorllms`两个指令来安装jittorllms的依赖（在项目根目录运行这两个指令）。" +\
r"警告：安装jittorllms依赖后将完全破坏现有的pytorch环境，建议使用docker环境！" + trimmed_format_exc()
self.success = False
@@ -45,15 +45,15 @@ class GetGLMHandle(Process):
env = os.environ.get("PATH", "")
os.environ["PATH"] = env.replace('/cuda/bin', '/x/bin')
root_dir_assume = os.path.abspath(os.path.dirname(__file__) + '/..')
- os.chdir(root_dir_assume + '/request_llms/jittorllms')
- sys.path.append(root_dir_assume + '/request_llms/jittorllms')
+ os.chdir(root_dir_assume + '/request_llm/jittorllms')
+ sys.path.append(root_dir_assume + '/request_llm/jittorllms')
validate_path() # validate path so you can run from base directory
def load_model():
import types
try:
if self.jittorllms_model is None:
- device = get_conf('LOCAL_MODEL_DEVICE')
+ device, = get_conf('LOCAL_MODEL_DEVICE')
from .jittorllms.models import get_model
# availabel_models = ["chatglm", "pangualpha", "llama", "chatrwkv"]
args_dict = {'model': 'pangualpha'}
@@ -109,7 +109,7 @@ pangu_glm_handle = None
def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="", observe_window=[], console_slience=False):
"""
多线程方法
- 函数的说明请见 request_llms/bridge_all.py
+ 函数的说明请见 request_llm/bridge_all.py
"""
global pangu_glm_handle
if pangu_glm_handle is None:
@@ -140,7 +140,7 @@ def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="",
def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_prompt='', stream = True, additional_fn=None):
"""
单线程方法
- 函数的说明请见 request_llms/bridge_all.py
+ 函数的说明请见 request_llm/bridge_all.py
"""
chatbot.append((inputs, ""))
@@ -163,13 +163,13 @@ def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_promp
history_feedin.append([history[2*i], history[2*i+1]] )
# 开始接收jittorllms的回复
- response = "[Local Message] 等待jittorllms响应中 ..."
+ response = "[Local Message]: 等待jittorllms响应中 ..."
for response in pangu_glm_handle.stream_chat(query=inputs, history=history_feedin, system_prompt=system_prompt, max_length=llm_kwargs['max_length'], top_p=llm_kwargs['top_p'], temperature=llm_kwargs['temperature']):
chatbot[-1] = (inputs, response)
yield from update_ui(chatbot=chatbot, history=history)
# 总结输出
- if response == "[Local Message] 等待jittorllms响应中 ...":
- response = "[Local Message] jittorllms响应异常 ..."
+ if response == "[Local Message]: 等待jittorllms响应中 ...":
+ response = "[Local Message]: jittorllms响应异常 ..."
history.extend([inputs, response])
yield from update_ui(chatbot=chatbot, history=history)
diff --git a/request_llms/bridge_jittorllms_rwkv.py b/request_llm/bridge_jittorllms_rwkv.py
similarity index 90%
rename from request_llms/bridge_jittorllms_rwkv.py
rename to request_llm/bridge_jittorllms_rwkv.py
index 0021a50d0ab1f6bb2a909003f1ea36ccffa326c7..ee4f592f5a1a3e6022b41c899e342bd7e55ed44f 100644
--- a/request_llms/bridge_jittorllms_rwkv.py
+++ b/request_llm/bridge_jittorllms_rwkv.py
@@ -28,8 +28,8 @@ class GetGLMHandle(Process):
self.success = True
except:
from toolbox import trimmed_format_exc
- self.info = r"缺少jittorllms的依赖，如果要使用jittorllms，除了基础的pip依赖以外，您还需要运行`pip install -r request_llms/requirements_jittorllms.txt -i https://pypi.jittor.org/simple -I`"+\
- r"和`git clone https://gitlink.org.cn/jittor/JittorLLMs.git --depth 1 request_llms/jittorllms`两个指令来安装jittorllms的依赖（在项目根目录运行这两个指令）。" +\
+ self.info = r"缺少jittorllms的依赖，如果要使用jittorllms，除了基础的pip依赖以外，您还需要运行`pip install -r request_llm/requirements_jittorllms.txt -i https://pypi.jittor.org/simple -I`"+\
+ r"和`git clone https://gitlink.org.cn/jittor/JittorLLMs.git --depth 1 request_llm/jittorllms`两个指令来安装jittorllms的依赖（在项目根目录运行这两个指令）。" +\
r"警告：安装jittorllms依赖后将完全破坏现有的pytorch环境，建议使用docker环境！" + trimmed_format_exc()
self.success = False
@@ -45,15 +45,15 @@ class GetGLMHandle(Process):
env = os.environ.get("PATH", "")
os.environ["PATH"] = env.replace('/cuda/bin', '/x/bin')
root_dir_assume = os.path.abspath(os.path.dirname(__file__) + '/..')
- os.chdir(root_dir_assume + '/request_llms/jittorllms')
- sys.path.append(root_dir_assume + '/request_llms/jittorllms')
+ os.chdir(root_dir_assume + '/request_llm/jittorllms')
+ sys.path.append(root_dir_assume + '/request_llm/jittorllms')
validate_path() # validate path so you can run from base directory
def load_model():
import types
try:
if self.jittorllms_model is None:
- device = get_conf('LOCAL_MODEL_DEVICE')
+ device, = get_conf('LOCAL_MODEL_DEVICE')
from .jittorllms.models import get_model
# availabel_models = ["chatglm", "pangualpha", "llama", "chatrwkv"]
args_dict = {'model': 'chatrwkv'}
@@ -109,7 +109,7 @@ rwkv_glm_handle = None
def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="", observe_window=[], console_slience=False):
"""
多线程方法
- 函数的说明请见 request_llms/bridge_all.py
+ 函数的说明请见 request_llm/bridge_all.py
"""
global rwkv_glm_handle
if rwkv_glm_handle is None:
@@ -140,7 +140,7 @@ def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="",
def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_prompt='', stream = True, additional_fn=None):
"""
单线程方法
- 函数的说明请见 request_llms/bridge_all.py
+ 函数的说明请见 request_llm/bridge_all.py
"""
chatbot.append((inputs, ""))
@@ -163,13 +163,13 @@ def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_promp
history_feedin.append([history[2*i], history[2*i+1]] )
# 开始接收jittorllms的回复
- response = "[Local Message] 等待jittorllms响应中 ..."
+ response = "[Local Message]: 等待jittorllms响应中 ..."
for response in rwkv_glm_handle.stream_chat(query=inputs, history=history_feedin, system_prompt=system_prompt, max_length=llm_kwargs['max_length'], top_p=llm_kwargs['top_p'], temperature=llm_kwargs['temperature']):
chatbot[-1] = (inputs, response)
yield from update_ui(chatbot=chatbot, history=history)
# 总结输出
- if response == "[Local Message] 等待jittorllms响应中 ...":
- response = "[Local Message] jittorllms响应异常 ..."
+ if response == "[Local Message]: 等待jittorllms响应中 ...":
+ response = "[Local Message]: jittorllms响应异常 ..."
history.extend([inputs, response])
yield from update_ui(chatbot=chatbot, history=history)
diff --git a/request_llms/bridge_llama2.py b/request_llm/bridge_llama2.py
similarity index 94%
rename from request_llms/bridge_llama2.py
rename to request_llm/bridge_llama2.py
index bfa3c14ae63cb1eafe85b6d81810b0f70303a900..e236c94272b110b2d4e4eb72812a270a0c8b3226 100644
--- a/request_llms/bridge_llama2.py
+++ b/request_llm/bridge_llama2.py
@@ -1,18 +1,19 @@
model_name = "LLaMA"
-cmd_to_install = "`pip install -r request_llms/requirements_chatglm.txt`"
+cmd_to_install = "`pip install -r request_llm/requirements_chatglm.txt`"
from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
from toolbox import update_ui, get_conf, ProxyNetworkActivate
from multiprocessing import Process, Pipe
-from .local_llm_class import LocalLLMHandle, get_local_llm_predict_fns
+from .local_llm_class import LocalLLMHandle, get_local_llm_predict_fns, SingletonLocalLLM
from threading import Thread
# ------------------------------------------------------------------------------------------------------------------------
# 🔌💻 Local Model
# ------------------------------------------------------------------------------------------------------------------------
-class GetLlamaHandle(LocalLLMHandle):
+@SingletonLocalLLM
+class GetONNXGLMHandle(LocalLLMHandle):
def load_model_info(self):
# 🏃♂️🏃♂️🏃♂️ 子进程执行
@@ -29,7 +30,7 @@ class GetLlamaHandle(LocalLLMHandle):
with open(os.path.expanduser('~/.cache/huggingface/token'), 'w') as f:
f.write(huggingface_token)
model_id = 'meta-llama/Llama-2-7b-chat-hf'
- with ProxyNetworkActivate('Download_LLM'):
+ with ProxyNetworkActivate():
self._tokenizer = AutoTokenizer.from_pretrained(model_id, use_auth_token=huggingface_token)
# use fp16
model = AutoModelForCausalLM.from_pretrained(model_id, use_auth_token=huggingface_token).eval()
@@ -87,4 +88,4 @@ class GetLlamaHandle(LocalLLMHandle):
# ------------------------------------------------------------------------------------------------------------------------
# 🔌💻 GPT-Academic Interface
# ------------------------------------------------------------------------------------------------------------------------
-predict_no_ui_long_connection, predict = get_local_llm_predict_fns(GetLlamaHandle, model_name)
\ No newline at end of file
+predict_no_ui_long_connection, predict = get_local_llm_predict_fns(GetONNXGLMHandle, model_name)
\ No newline at end of file
diff --git a/request_llms/bridge_moss.py b/request_llm/bridge_moss.py
similarity index 93%
rename from request_llms/bridge_moss.py
rename to request_llm/bridge_moss.py
index ee8907cf01a6958b92d5abeed3487ee9f21524f9..3c6217d2b285c499490d81e9a744b2dd6f485e24 100644
--- a/request_llms/bridge_moss.py
+++ b/request_llm/bridge_moss.py
@@ -1,6 +1,8 @@
+from transformers import AutoModel, AutoTokenizer
import time
import threading
+import importlib
from toolbox import update_ui, get_conf
from multiprocessing import Process, Pipe
@@ -22,12 +24,12 @@ class GetGLMHandle(Process):
def check_dependency(self): # 主进程执行
try:
import datasets, os
- assert os.path.exists('request_llms/moss/models')
+ assert os.path.exists('request_llm/moss/models')
self.info = "依赖检测通过"
self.success = True
except:
self.info = """
- 缺少MOSS的依赖，如果要使用MOSS，除了基础的pip依赖以外，您还需要运行`pip install -r request_llms/requirements_moss.txt`和`git clone https://github.com/OpenLMLab/MOSS.git request_llms/moss`安装MOSS的依赖。
+ 缺少MOSS的依赖，如果要使用MOSS，除了基础的pip依赖以外，您还需要运行`pip install -r request_llm/requirements_moss.txt`和`git clone https://github.com/OpenLMLab/MOSS.git request_llm/moss`安装MOSS的依赖。
"""
self.success = False
return self.success
@@ -108,8 +110,8 @@ class GetGLMHandle(Process):
def validate_path():
import os, sys
root_dir_assume = os.path.abspath(os.path.dirname(__file__) + '/..')
- os.chdir(root_dir_assume + '/request_llms/moss')
- sys.path.append(root_dir_assume + '/request_llms/moss')
+ os.chdir(root_dir_assume + '/request_llm/moss')
+ sys.path.append(root_dir_assume + '/request_llm/moss')
validate_path() # validate path so you can run from base directory
try:
@@ -174,7 +176,7 @@ moss_handle = None
def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="", observe_window=[], console_slience=False):
"""
多线程方法
- 函数的说明请见 request_llms/bridge_all.py
+ 函数的说明请见 request_llm/bridge_all.py
"""
global moss_handle
if moss_handle is None:
@@ -204,7 +206,7 @@ def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="",
def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_prompt='', stream = True, additional_fn=None):
"""
单线程方法
- 函数的说明请见 request_llms/bridge_all.py
+ 函数的说明请见 request_llm/bridge_all.py
"""
chatbot.append((inputs, ""))
@@ -217,7 +219,7 @@ def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_promp
moss_handle = None
return
else:
- response = "[Local Message] 等待MOSS响应中 ..."
+ response = "[Local Message]: 等待MOSS响应中 ..."
chatbot[-1] = (inputs, response)
yield from update_ui(chatbot=chatbot, history=history)
@@ -236,7 +238,7 @@ def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_promp
yield from update_ui(chatbot=chatbot, history=history)
# 总结输出
- if response == "[Local Message] 等待MOSS响应中 ...":
- response = "[Local Message] MOSS响应异常 ..."
+ if response == "[Local Message]: 等待MOSS响应中 ...":
+ response = "[Local Message]: MOSS响应异常 ..."
history.extend([inputs, response.strip('<|MOSS|>: ')])
yield from update_ui(chatbot=chatbot, history=history)
diff --git a/request_llm/bridge_newbing.py b/request_llm/bridge_newbing.py
new file mode 100644
index 0000000000000000000000000000000000000000..2136f01beb3edd25b94dd8048c20b63a14ef905e
--- /dev/null
+++ b/request_llm/bridge_newbing.py
@@ -0,0 +1,254 @@
+"""
+========================================================================
+第一部分：来自EdgeGPT.py
+https://github.com/acheong08/EdgeGPT
+========================================================================
+"""
+from .edge_gpt import NewbingChatbot
+load_message = "等待NewBing响应。"
+
+"""
+========================================================================
+第二部分：子进程Worker（调用主体）
+========================================================================
+"""
+import time
+import json
+import re
+import logging
+import asyncio
+import importlib
+import threading
+from toolbox import update_ui, get_conf, trimmed_format_exc
+from multiprocessing import Process, Pipe
+
+def preprocess_newbing_out(s):
+ pattern = r'\^(\d+)\^' # 匹配^数字^
+ sub = lambda m: '('+m.group(1)+')' # 将匹配到的数字作为替换值
+ result = re.sub(pattern, sub, s) # 替换操作
+ if '[1]' in result:
+ result += '\n\n```reference\n' + "\n".join([r for r in result.split('\n') if r.startswith('[')]) + '\n```\n'
+ return result
+
+def preprocess_newbing_out_simple(result):
+ if '[1]' in result:
+ result += '\n\n```reference\n' + "\n".join([r for r in result.split('\n') if r.startswith('[')]) + '\n```\n'
+ return result
+
+class NewBingHandle(Process):
+ def __init__(self):
+ super().__init__(daemon=True)
+ self.parent, self.child = Pipe()
+ self.newbing_model = None
+ self.info = ""
+ self.success = True
+ self.local_history = []
+ self.check_dependency()
+ self.start()
+ self.threadLock = threading.Lock()
+
+ def check_dependency(self):
+ try:
+ self.success = False
+ import certifi, httpx, rich
+ self.info = "依赖检测通过，等待NewBing响应。注意目前不能多人同时调用NewBing接口（有线程锁），否则将导致每个人的NewBing问询历史互相渗透。调用NewBing时，会自动使用已配置的代理。"
+ self.success = True
+ except:
+ self.info = "缺少的依赖，如果要使用Newbing，除了基础的pip依赖以外，您还需要运行`pip install -r request_llm/requirements_newbing.txt`安装Newbing的依赖。"
+ self.success = False
+
+ def ready(self):
+ return self.newbing_model is not None
+
+ async def async_run(self):
+ # 读取配置
+ NEWBING_STYLE, = get_conf('NEWBING_STYLE')
+ from request_llm.bridge_all import model_info
+ endpoint = model_info['newbing']['endpoint']
+ while True:
+ # 等待
+ kwargs = self.child.recv()
+ question=kwargs['query']
+ history=kwargs['history']
+ system_prompt=kwargs['system_prompt']
+
+ # 是否重置
+ if len(self.local_history) > 0 and len(history)==0:
+ await self.newbing_model.reset()
+ self.local_history = []
+
+ # 开始问问题
+ prompt = ""
+ if system_prompt not in self.local_history:
+ self.local_history.append(system_prompt)
+ prompt += system_prompt + '\n'
+
+ # 追加历史
+ for ab in history:
+ a, b = ab
+ if a not in self.local_history:
+ self.local_history.append(a)
+ prompt += a + '\n'
+ # if b not in self.local_history:
+ # self.local_history.append(b)
+ # prompt += b + '\n'
+
+ # 问题
+ prompt += question
+ self.local_history.append(question)
+ print('question:', prompt)
+ # 提交
+ async for final, response in self.newbing_model.ask_stream(
+ prompt=question,
+ conversation_style=NEWBING_STYLE, # ["creative", "balanced", "precise"]
+ wss_link=endpoint, # "wss://sydney.bing.com/sydney/ChatHub"
+ ):
+ if not final:
+ print(response)
+ self.child.send(str(response))
+ else:
+ print('-------- receive final ---------')
+ self.child.send('[Finish]')
+ # self.local_history.append(response)
+
+
+ def run(self):
+ """
+ 这个函数运行在子进程
+ """
+ # 第一次运行，加载参数
+ self.success = False
+ self.local_history = []
+ if (self.newbing_model is None) or (not self.success):
+ # 代理设置
+ proxies, = get_conf('proxies')
+ if proxies is None:
+ self.proxies_https = None
+ else:
+ self.proxies_https = proxies['https']
+ # cookie
+ NEWBING_COOKIES, = get_conf('NEWBING_COOKIES')
+ try:
+ cookies = json.loads(NEWBING_COOKIES)
+ except:
+ self.success = False
+ tb_str = '\n```\n' + trimmed_format_exc() + '\n```\n'
+ self.child.send(f'[Local Message] 不能加载Newbing组件。NEWBING_COOKIES未填写或有格式错误。')
+ self.child.send('[Fail]')
+ self.child.send('[Finish]')
+ raise RuntimeError(f"不能加载Newbing组件。NEWBING_COOKIES未填写或有格式错误。")
+
+ try:
+ self.newbing_model = NewbingChatbot(proxy=self.proxies_https, cookies=cookies)
+ except:
+ self.success = False
+ tb_str = '\n```\n' + trimmed_format_exc() + '\n```\n'
+ self.child.send(f'[Local Message] 不能加载Newbing组件。{tb_str}')
+ self.child.send('[Fail]')
+ self.child.send('[Finish]')
+ raise RuntimeError(f"不能加载Newbing组件。")
+
+ self.success = True
+ try:
+ # 进入任务等待状态
+ asyncio.run(self.async_run())
+ except Exception:
+ tb_str = '\n```\n' + trimmed_format_exc() + '\n```\n'
+ self.child.send(f'[Local Message] Newbing失败 {tb_str}.')
+ self.child.send('[Fail]')
+ self.child.send('[Finish]')
+
+ def stream_chat(self, **kwargs):
+ """
+ 这个函数运行在主进程
+ """
+ self.threadLock.acquire()
+ self.parent.send(kwargs) # 发送请求到子进程
+ while True:
+ res = self.parent.recv() # 等待newbing回复的片段
+ if res == '[Finish]':
+ break # 结束
+ elif res == '[Fail]':
+ self.success = False
+ break
+ else:
+ yield res # newbing回复的片段
+ self.threadLock.release()
+
+
+"""
+========================================================================
+第三部分：主进程统一调用函数接口
+========================================================================
+"""
+global newbing_handle
+newbing_handle = None
+
+def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="", observe_window=None, console_slience=False):
+ """
+ 多线程方法
+ 函数的说明请见 request_llm/bridge_all.py
+ """
+ global newbing_handle
+ if (newbing_handle is None) or (not newbing_handle.success):
+ newbing_handle = NewBingHandle()
+ observe_window[0] = load_message + "\n\n" + newbing_handle.info
+ if not newbing_handle.success:
+ error = newbing_handle.info
+ newbing_handle = None
+ raise RuntimeError(error)
+
+ # 没有 sys_prompt 接口，因此把prompt加入 history
+ history_feedin = []
+ for i in range(len(history)//2):
+ history_feedin.append([history[2*i], history[2*i+1]] )
+
+ watch_dog_patience = 5 # 看门狗 (watchdog) 的耐心, 设置5秒即可
+ response = ""
+ observe_window[0] = "[Local Message]: 等待NewBing响应中 ..."
+ for response in newbing_handle.stream_chat(query=inputs, history=history_feedin, system_prompt=sys_prompt, max_length=llm_kwargs['max_length'], top_p=llm_kwargs['top_p'], temperature=llm_kwargs['temperature']):
+ observe_window[0] = preprocess_newbing_out_simple(response)
+ if len(observe_window) >= 2:
+ if (time.time()-observe_window[1]) > watch_dog_patience:
+ raise RuntimeError("程序终止。")
+ return preprocess_newbing_out_simple(response)
+
+def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_prompt='', stream = True, additional_fn=None):
+ """
+ 单线程方法
+ 函数的说明请见 request_llm/bridge_all.py
+ """
+ chatbot.append((inputs, "[Local Message]: 等待NewBing响应中 ..."))
+
+ global newbing_handle
+ if (newbing_handle is None) or (not newbing_handle.success):
+ newbing_handle = NewBingHandle()
+ chatbot[-1] = (inputs, load_message + "\n\n" + newbing_handle.info)
+ yield from update_ui(chatbot=chatbot, history=[])
+ if not newbing_handle.success:
+ newbing_handle = None
+ return
+
+ if additional_fn is not None:
+ import core_functional
+ importlib.reload(core_functional) # 热更新prompt
+ core_functional = core_functional.get_core_functions()
+ if "PreProcess" in core_functional[additional_fn]: inputs = core_functional[additional_fn]["PreProcess"](inputs) # 获取预处理函数（如果有的话）
+ inputs = core_functional[additional_fn]["Prefix"] + inputs + core_functional[additional_fn]["Suffix"]
+
+ history_feedin = []
+ for i in range(len(history)//2):
+ history_feedin.append([history[2*i], history[2*i+1]] )
+
+ chatbot[-1] = (inputs, "[Local Message]: 等待NewBing响应中 ...")
+ response = "[Local Message]: 等待NewBing响应中 ..."
+ yield from update_ui(chatbot=chatbot, history=history, msg="NewBing响应缓慢，尚未完成全部响应，请耐心完成后再提交新问题。")
+ for response in newbing_handle.stream_chat(query=inputs, history=history_feedin, system_prompt=system_prompt, max_length=llm_kwargs['max_length'], top_p=llm_kwargs['top_p'], temperature=llm_kwargs['temperature']):
+ chatbot[-1] = (inputs, preprocess_newbing_out(response))
+ yield from update_ui(chatbot=chatbot, history=history, msg="NewBing响应缓慢，尚未完成全部响应，请耐心完成后再提交新问题。")
+ if response == "[Local Message]: 等待NewBing响应中 ...": response = "[Local Message]: NewBing响应异常，请刷新界面重试 ..."
+ history.extend([inputs, response])
+ logging.info(f'[raw_input] {inputs}')
+ logging.info(f'[response] {response}')
+ yield from update_ui(chatbot=chatbot, history=history, msg="完成全部响应，请提交新问题。")
+
diff --git a/request_llm/bridge_newbingfree.py b/request_llm/bridge_newbingfree.py
new file mode 100644
index 0000000000000000000000000000000000000000..cc6e9b733b48fe255c15bc9fae3c9abd74f276ca
--- /dev/null
+++ b/request_llm/bridge_newbingfree.py
@@ -0,0 +1,245 @@
+"""
+========================================================================
+第一部分：来自EdgeGPT.py
+https://github.com/acheong08/EdgeGPT
+========================================================================
+"""
+from .edge_gpt_free import Chatbot as NewbingChatbot
+load_message = "等待NewBing响应。"
+
+"""
+========================================================================
+第二部分：子进程Worker（调用主体）
+========================================================================
+"""
+import time
+import json
+import re
+import logging
+import asyncio
+import importlib
+import threading
+from toolbox import update_ui, get_conf, trimmed_format_exc
+from multiprocessing import Process, Pipe
+
+def preprocess_newbing_out(s):
+ pattern = r'\^(\d+)\^' # 匹配^数字^
+ sub = lambda m: '('+m.group(1)+')' # 将匹配到的数字作为替换值
+ result = re.sub(pattern, sub, s) # 替换操作
+ if '[1]' in result:
+ result += '\n\n```reference\n' + "\n".join([r for r in result.split('\n') if r.startswith('[')]) + '\n```\n'
+ return result
+
+def preprocess_newbing_out_simple(result):
+ if '[1]' in result:
+ result += '\n\n```reference\n' + "\n".join([r for r in result.split('\n') if r.startswith('[')]) + '\n```\n'
+ return result
+
+class NewBingHandle(Process):
+ def __init__(self):
+ super().__init__(daemon=True)
+ self.parent, self.child = Pipe()
+ self.newbing_model = None
+ self.info = ""
+ self.success = True
+ self.local_history = []
+ self.check_dependency()
+ self.start()
+ self.threadLock = threading.Lock()
+
+ def check_dependency(self):
+ try:
+ self.success = False
+ import certifi, httpx, rich
+ self.info = "依赖检测通过，等待NewBing响应。注意目前不能多人同时调用NewBing接口（有线程锁），否则将导致每个人的NewBing问询历史互相渗透。调用NewBing时，会自动使用已配置的代理。"
+ self.success = True
+ except:
+ self.info = "缺少的依赖，如果要使用Newbing，除了基础的pip依赖以外，您还需要运行`pip install -r request_llm/requirements_newbing.txt`安装Newbing的依赖。"
+ self.success = False
+
+ def ready(self):
+ return self.newbing_model is not None
+
+ async def async_run(self):
+ # 读取配置
+ NEWBING_STYLE, = get_conf('NEWBING_STYLE')
+ from request_llm.bridge_all import model_info
+ endpoint = model_info['newbing']['endpoint']
+ while True:
+ # 等待
+ kwargs = self.child.recv()
+ question=kwargs['query']
+ history=kwargs['history']
+ system_prompt=kwargs['system_prompt']
+
+ # 是否重置
+ if len(self.local_history) > 0 and len(history)==0:
+ await self.newbing_model.reset()
+ self.local_history = []
+
+ # 开始问问题
+ prompt = ""
+ if system_prompt not in self.local_history:
+ self.local_history.append(system_prompt)
+ prompt += system_prompt + '\n'
+
+ # 追加历史
+ for ab in history:
+ a, b = ab
+ if a not in self.local_history:
+ self.local_history.append(a)
+ prompt += a + '\n'
+
+ # 问题
+ prompt += question
+ self.local_history.append(question)
+ print('question:', prompt)
+ # 提交
+ async for final, response in self.newbing_model.ask_stream(
+ prompt=question,
+ conversation_style=NEWBING_STYLE, # ["creative", "balanced", "precise"]
+ wss_link=endpoint, # "wss://sydney.bing.com/sydney/ChatHub"
+ ):
+ if not final:
+ print(response)
+ self.child.send(str(response))
+ else:
+ print('-------- receive final ---------')
+ self.child.send('[Finish]')
+ # self.local_history.append(response)
+
+
+ def run(self):
+ """
+ 这个函数运行在子进程
+ """
+ # 第一次运行，加载参数
+ self.success = False
+ self.local_history = []
+ if (self.newbing_model is None) or (not self.success):
+ # 代理设置
+ proxies, NEWBING_COOKIES = get_conf('proxies', 'NEWBING_COOKIES')
+ if proxies is None:
+ self.proxies_https = None
+ else:
+ self.proxies_https = proxies['https']
+
+ if (NEWBING_COOKIES is not None) and len(NEWBING_COOKIES) > 100:
+ try:
+ cookies = json.loads(NEWBING_COOKIES)
+ except:
+ self.success = False
+ tb_str = '\n```\n' + trimmed_format_exc() + '\n```\n'
+ self.child.send(f'[Local Message] NEWBING_COOKIES未填写或有格式错误。')
+ self.child.send('[Fail]'); self.child.send('[Finish]')
+ raise RuntimeError(f"NEWBING_COOKIES未填写或有格式错误。")
+ else:
+ cookies = None
+
+ try:
+ self.newbing_model = NewbingChatbot(proxy=self.proxies_https, cookies=cookies)
+ except:
+ self.success = False
+ tb_str = '\n```\n' + trimmed_format_exc() + '\n```\n'
+ self.child.send(f'[Local Message] 不能加载Newbing组件。{tb_str}')
+ self.child.send('[Fail]')
+ self.child.send('[Finish]')
+ raise RuntimeError(f"不能加载Newbing组件。")
+
+ self.success = True
+ try:
+ # 进入任务等待状态
+ asyncio.run(self.async_run())
+ except Exception:
+ tb_str = '\n```\n' + trimmed_format_exc() + '\n```\n'
+ self.child.send(f'[Local Message] Newbing 请求失败，报错信息如下. 如果是与网络相关的问题，建议更换代理协议（推荐http）或代理节点 {tb_str}.')
+ self.child.send('[Fail]')
+ self.child.send('[Finish]')
+
+ def stream_chat(self, **kwargs):
+ """
+ 这个函数运行在主进程
+ """
+ self.threadLock.acquire() # 获取线程锁
+ self.parent.send(kwargs) # 请求子进程
+ while True:
+ res = self.parent.recv() # 等待newbing回复的片段
+ if res == '[Finish]': break # 结束
+ elif res == '[Fail]': self.success = False; break # 失败
+ else: yield res # newbing回复的片段
+ self.threadLock.release() # 释放线程锁
+
+
+"""
+========================================================================
+第三部分：主进程统一调用函数接口
+========================================================================
+"""
+global newbingfree_handle
+newbingfree_handle = None
+
+def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="", observe_window=[], console_slience=False):
+ """
+ 多线程方法
+ 函数的说明请见 request_llm/bridge_all.py
+ """
+ global newbingfree_handle
+ if (newbingfree_handle is None) or (not newbingfree_handle.success):
+ newbingfree_handle = NewBingHandle()
+ if len(observe_window) >= 1: observe_window[0] = load_message + "\n\n" + newbingfree_handle.info
+ if not newbingfree_handle.success:
+ error = newbingfree_handle.info
+ newbingfree_handle = None
+ raise RuntimeError(error)
+
+ # 没有 sys_prompt 接口，因此把prompt加入 history
+ history_feedin = []
+ for i in range(len(history)//2):
+ history_feedin.append([history[2*i], history[2*i+1]] )
+
+ watch_dog_patience = 5 # 看门狗 (watchdog) 的耐心, 设置5秒即可
+ response = ""
+ if len(observe_window) >= 1: observe_window[0] = "[Local Message]: 等待NewBing响应中 ..."
+ for response in newbingfree_handle.stream_chat(query=inputs, history=history_feedin, system_prompt=sys_prompt, max_length=llm_kwargs['max_length'], top_p=llm_kwargs['top_p'], temperature=llm_kwargs['temperature']):
+ if len(observe_window) >= 1: observe_window[0] = preprocess_newbing_out_simple(response)
+ if len(observe_window) >= 2:
+ if (time.time()-observe_window[1]) > watch_dog_patience:
+ raise RuntimeError("程序终止。")
+ return preprocess_newbing_out_simple(response)
+
+def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_prompt='', stream = True, additional_fn=None):
+ """
+ 单线程方法
+ 函数的说明请见 request_llm/bridge_all.py
+ """
+ chatbot.append((inputs, "[Local Message]: 等待NewBing响应中 ..."))
+
+ global newbingfree_handle
+ if (newbingfree_handle is None) or (not newbingfree_handle.success):
+ newbingfree_handle = NewBingHandle()
+ chatbot[-1] = (inputs, load_message + "\n\n" + newbingfree_handle.info)
+ yield from update_ui(chatbot=chatbot, history=[])
+ if not newbingfree_handle.success:
+ newbingfree_handle = None
+ return
+
+ if additional_fn is not None:
+ from core_functional import handle_core_functionality
+ inputs, history = handle_core_functionality(additional_fn, inputs, history, chatbot)
+
+ history_feedin = []
+ for i in range(len(history)//2):
+ history_feedin.append([history[2*i], history[2*i+1]] )
+
+ chatbot[-1] = (inputs, "[Local Message]: 等待NewBing响应中 ...")
+ response = "[Local Message]: 等待NewBing响应中 ..."
+ yield from update_ui(chatbot=chatbot, history=history, msg="NewBing响应缓慢，尚未完成全部响应，请耐心完成后再提交新问题。")
+ for response in newbingfree_handle.stream_chat(query=inputs, history=history_feedin, system_prompt=system_prompt, max_length=llm_kwargs['max_length'], top_p=llm_kwargs['top_p'], temperature=llm_kwargs['temperature']):
+ chatbot[-1] = (inputs, preprocess_newbing_out(response))
+ yield from update_ui(chatbot=chatbot, history=history, msg="NewBing响应缓慢，尚未完成全部响应，请耐心完成后再提交新问题。")
+ if response == "[Local Message]: 等待NewBing响应中 ...": response = "[Local Message]: NewBing响应异常，请刷新界面重试 ..."
+ history.extend([inputs, response])
+ logging.info(f'[raw_input] {inputs}')
+ logging.info(f'[response] {response}')
+ yield from update_ui(chatbot=chatbot, history=history, msg="完成全部响应，请提交新问题。")
+
diff --git a/request_llms/bridge_qianfan.py b/request_llm/bridge_qianfan.py
similarity index 91%
rename from request_llms/bridge_qianfan.py
rename to request_llm/bridge_qianfan.py
index 0f02457b23847e32efd10726a65d719d904e6aa5..be7397607a71681edc240be48d1889ce51008af5 100644
--- a/request_llms/bridge_qianfan.py
+++ b/request_llm/bridge_qianfan.py
@@ -75,12 +75,11 @@ def generate_message_payload(inputs, llm_kwargs, history, system_prompt):
def generate_from_baidu_qianfan(inputs, llm_kwargs, history, system_prompt):
- BAIDU_CLOUD_QIANFAN_MODEL = get_conf('BAIDU_CLOUD_QIANFAN_MODEL')
+ BAIDU_CLOUD_QIANFAN_MODEL, = get_conf('BAIDU_CLOUD_QIANFAN_MODEL')
url_lib = {
- "ERNIE-Bot-4": "https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/completions_pro",
- "ERNIE-Bot": "https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/completions",
- "ERNIE-Bot-turbo": "https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/eb-instant",
+ "ERNIE-Bot": "https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/completions" ,
+ "ERNIE-Bot-turbo": "https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/eb-instant" ,
"BLOOMZ-7B": "https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/bloomz_7b1",
"Llama-2-70B-Chat": "https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/llama_2_70b",
@@ -120,7 +119,7 @@ def generate_from_baidu_qianfan(inputs, llm_kwargs, history, system_prompt):
def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="", observe_window=[], console_slience=False):
"""
⭐多线程方法
- 函数的说明请见 request_llms/bridge_all.py
+ 函数的说明请见 request_llm/bridge_all.py
"""
watch_dog_patience = 5
response = ""
@@ -135,7 +134,7 @@ def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="",
def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_prompt='', stream = True, additional_fn=None):
"""
⭐单线程方法
- 函数的说明请见 request_llms/bridge_all.py
+ 函数的说明请见 request_llm/bridge_all.py
"""
chatbot.append((inputs, ""))
@@ -146,17 +145,21 @@ def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_promp
yield from update_ui(chatbot=chatbot, history=history)
# 开始接收回复
try:
- response = f"[Local Message] 等待{model_name}响应中 ..."
for response in generate_from_baidu_qianfan(inputs, llm_kwargs, history, system_prompt):
chatbot[-1] = (inputs, response)
yield from update_ui(chatbot=chatbot, history=history)
- history.extend([inputs, response])
- yield from update_ui(chatbot=chatbot, history=history)
except ConnectionAbortedError as e:
from .bridge_all import model_info
if len(history) >= 2: history[-1] = ""; history[-2] = "" # 清除当前溢出的输入：history[-2] 是本次输入, history[-1] 是本次输出
- history = clip_history(inputs=inputs, history=history, tokenizer=model_info[llm_kwargs['llm_model']]['tokenizer'],
+ history = clip_history(inputs=inputs, history=history, tokenizer=model_info[llm_kwargs['llm_model']]['tokenizer'],
max_token_limit=(model_info[llm_kwargs['llm_model']]['max_token'])) # history至少释放二分之一
chatbot[-1] = (chatbot[-1][0], "[Local Message] Reduce the length. 本次输入过长, 或历史数据过长. 历史缓存数据已部分释放, 您可以请再次尝试. (若再次失败则更可能是因为输入过长.)")
yield from update_ui(chatbot=chatbot, history=history, msg="异常") # 刷新界面
return
+
+ # 总结输出
+ response = f"[Local Message]: {model_name}响应异常 ..."
+ if response == f"[Local Message]: 等待{model_name}响应中 ...":
+ response = f"[Local Message]: {model_name}响应异常 ..."
+ history.extend([inputs, response])
+ yield from update_ui(chatbot=chatbot, history=history)
\ No newline at end of file
diff --git a/request_llms/bridge_qwen_local.py b/request_llm/bridge_qwen.py
similarity index 61%
rename from request_llms/bridge_qwen_local.py
rename to request_llm/bridge_qwen.py
index e6c2dd5cdf7b6ba59b474b3c38afebc77014dd14..07ed243feb25274213ccee90f862662080809f84 100644
--- a/request_llms/bridge_qwen_local.py
+++ b/request_llm/bridge_qwen.py
@@ -1,15 +1,22 @@
-model_name = "Qwen_Local"
-cmd_to_install = "`pip install -r request_llms/requirements_qwen_local.txt`"
+model_name = "Qwen"
+cmd_to_install = "`pip install -r request_llm/requirements_qwen.txt`"
-from toolbox import ProxyNetworkActivate, get_conf
-from .local_llm_class import LocalLLMHandle, get_local_llm_predict_fns
+
+from transformers import AutoModel, AutoTokenizer
+import time
+import threading
+import importlib
+from toolbox import update_ui, get_conf
+from multiprocessing import Process, Pipe
+from .local_llm_class import LocalLLMHandle, get_local_llm_predict_fns, SingletonLocalLLM
# ------------------------------------------------------------------------------------------------------------------------
# 🔌💻 Local Model
# ------------------------------------------------------------------------------------------------------------------------
-class GetQwenLMHandle(LocalLLMHandle):
+@SingletonLocalLLM
+class GetONNXGLMHandle(LocalLLMHandle):
def load_model_info(self):
# 🏃♂️🏃♂️🏃♂️ 子进程执行
@@ -18,16 +25,18 @@ class GetQwenLMHandle(LocalLLMHandle):
def load_model_and_tokenizer(self):
# 🏃♂️🏃♂️🏃♂️ 子进程执行
- # from modelscope import AutoModelForCausalLM, AutoTokenizer, GenerationConfig
- from transformers import AutoModelForCausalLM, AutoTokenizer
- from transformers.generation import GenerationConfig
- with ProxyNetworkActivate('Download_LLM'):
- model_id = get_conf('QWEN_LOCAL_MODEL_SELECTION')
- self._tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True, resume_download=True)
- # use fp16
- model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", trust_remote_code=True).eval()
- model.generation_config = GenerationConfig.from_pretrained(model_id, trust_remote_code=True) # 可指定不同的生成长度、top_p等相关超参
- self._model = model
+ import os, glob
+ import os
+ import platform
+ from modelscope import AutoModelForCausalLM, AutoTokenizer, GenerationConfig
+
+ model_id = 'qwen/Qwen-7B-Chat'
+ revision = 'v1.0.1'
+ self._tokenizer = AutoTokenizer.from_pretrained(model_id, revision=revision, trust_remote_code=True)
+ # use fp16
+ model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", revision=revision, trust_remote_code=True, fp16=True).eval()
+ model.generation_config = GenerationConfig.from_pretrained(model_id, trust_remote_code=True) # 可指定不同的生成长度、top_p等相关超参
+ self._model = model
return self._model, self._tokenizer
@@ -43,7 +52,7 @@ class GetQwenLMHandle(LocalLLMHandle):
query, max_length, top_p, temperature, history = adaptor(kwargs)
- for response in self._model.chat_stream(self._tokenizer, query, history=history):
+ for response in self._model.chat(self._tokenizer, query, history=history, stream=True):
yield response
def try_to_import_special_deps(self, **kwargs):
@@ -56,4 +65,4 @@ class GetQwenLMHandle(LocalLLMHandle):
# ------------------------------------------------------------------------------------------------------------------------
# 🔌💻 GPT-Academic Interface
# ------------------------------------------------------------------------------------------------------------------------
-predict_no_ui_long_connection, predict = get_local_llm_predict_fns(GetQwenLMHandle, model_name)
\ No newline at end of file
+predict_no_ui_long_connection, predict = get_local_llm_predict_fns(GetONNXGLMHandle, model_name)
\ No newline at end of file
diff --git a/request_llms/bridge_spark.py b/request_llm/bridge_spark.py
similarity index 75%
rename from request_llms/bridge_spark.py
rename to request_llm/bridge_spark.py
index 8449494c13a60d736058616b4264c589dbb35430..0fe925f7a0354fe6361e9d11ae074dd287813e9f 100644
--- a/request_llms/bridge_spark.py
+++ b/request_llm/bridge_spark.py
@@ -8,15 +8,15 @@ from multiprocessing import Process, Pipe
model_name = '星火认知大模型'
def validate_key():
- XFYUN_APPID = get_conf('XFYUN_APPID')
- if XFYUN_APPID == '00000000' or XFYUN_APPID == '':
+ XFYUN_APPID, = get_conf('XFYUN_APPID', )
+ if XFYUN_APPID == '00000000' or XFYUN_APPID == '':
return False
return True
def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="", observe_window=[], console_slience=False):
"""
⭐多线程方法
- 函数的说明请见 request_llms/bridge_all.py
+ 函数的说明请见 request_llm/bridge_all.py
"""
watch_dog_patience = 5
response = ""
@@ -26,7 +26,7 @@ def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="",
from .com_sparkapi import SparkRequestInstance
sri = SparkRequestInstance()
- for response in sri.generate(inputs, llm_kwargs, history, sys_prompt, use_image_api=False):
+ for response in sri.generate(inputs, llm_kwargs, history, sys_prompt):
if len(observe_window) >= 1:
observe_window[0] = response
if len(observe_window) >= 2:
@@ -36,29 +36,28 @@ def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="",
def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_prompt='', stream = True, additional_fn=None):
"""
⭐单线程方法
- 函数的说明请见 request_llms/bridge_all.py
+ 函数的说明请见 request_llm/bridge_all.py
"""
chatbot.append((inputs, ""))
yield from update_ui(chatbot=chatbot, history=history)
if validate_key() is False:
- yield from update_ui_lastest_msg(lastmsg="[Local Message] 请配置讯飞星火大模型的XFYUN_APPID, XFYUN_API_KEY, XFYUN_API_SECRET", chatbot=chatbot, history=history, delay=0)
+ yield from update_ui_lastest_msg(lastmsg="[Local Message]: 请配置讯飞星火大模型的XFYUN_APPID, XFYUN_API_KEY, XFYUN_API_SECRET", chatbot=chatbot, history=history, delay=0)
return
if additional_fn is not None:
from core_functional import handle_core_functionality
inputs, history = handle_core_functionality(additional_fn, inputs, history, chatbot)
- # 开始接收回复
+ # 开始接收回复
from .com_sparkapi import SparkRequestInstance
sri = SparkRequestInstance()
- response = f"[Local Message] 等待{model_name}响应中 ..."
- for response in sri.generate(inputs, llm_kwargs, history, system_prompt, use_image_api=True):
+ for response in sri.generate(inputs, llm_kwargs, history, system_prompt):
chatbot[-1] = (inputs, response)
yield from update_ui(chatbot=chatbot, history=history)
# 总结输出
- if response == f"[Local Message] 等待{model_name}响应中 ...":
- response = f"[Local Message] {model_name}响应异常 ..."
+ if response == f"[Local Message]: 等待{model_name}响应中 ...":
+ response = f"[Local Message]: {model_name}响应异常 ..."
history.extend([inputs, response])
yield from update_ui(chatbot=chatbot, history=history)
\ No newline at end of file
diff --git a/request_llms/bridge_stackclaude.py b/request_llm/bridge_stackclaude.py
similarity index 53%
rename from request_llms/bridge_stackclaude.py
rename to request_llm/bridge_stackclaude.py
index 21590b8a0853e169d4f9bcf2397d6bb487f19558..3f2ee67428f9c8323eca0f7006ad4d4f767a6b58 100644
--- a/request_llms/bridge_stackclaude.py
+++ b/request_llm/bridge_stackclaude.py
@@ -7,15 +7,14 @@ import logging
import time
from toolbox import get_conf
import asyncio
-
load_message = "正在加载Claude组件，请稍候..."
try:
"""
- =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+ ========================================================================
第一部分：Slack API Client
https://github.com/yokonsan/claude-in-slack-api
- =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+ ========================================================================
"""
from slack_sdk.errors import SlackApiError
@@ -24,23 +23,20 @@ try:
class SlackClient(AsyncWebClient):
"""SlackClient类用于与Slack API进行交互，实现消息发送、接收等功能。
- 属性：
- - CHANNEL_ID：str类型，表示频道ID。
+ 属性：
+ - CHANNEL_ID：str类型，表示频道ID。
- 方法：
- - open_channel()：异步方法。通过调用conversations_open方法打开一个频道，并将返回的频道ID保存在属性CHANNEL_ID中。
- - chat(text: str)：异步方法。向已打开的频道发送一条文本消息。
- - get_slack_messages()：异步方法。获取已打开频道的最新消息并返回消息列表，目前不支持历史消息查询。
- - get_reply()：异步方法。循环监听已打开频道的消息，如果收到"Typing…_"结尾的消息说明Claude还在继续输出，否则结束循环。
+ 方法：
+ - open_channel()：异步方法。通过调用conversations_open方法打开一个频道，并将返回的频道ID保存在属性CHANNEL_ID中。
+ - chat(text: str)：异步方法。向已打开的频道发送一条文本消息。
+ - get_slack_messages()：异步方法。获取已打开频道的最新消息并返回消息列表，目前不支持历史消息查询。
+ - get_reply()：异步方法。循环监听已打开频道的消息，如果收到"Typing…_"结尾的消息说明Claude还在继续输出，否则结束循环。
"""
-
CHANNEL_ID = None
async def open_channel(self):
- response = await self.conversations_open(
- users=get_conf("SLACK_CLAUDE_BOT_ID")
- )
+ response = await self.conversations_open(users=get_conf('SLACK_CLAUDE_BOT_ID')[0])
self.CHANNEL_ID = response["channel"]["id"]
async def chat(self, text):
@@ -53,39 +49,33 @@ try:
async def get_slack_messages(self):
try:
# TODO：暂时不支持历史消息，因为在同一个频道里存在多人使用时历史消息渗透问题
- resp = await self.conversations_history(
- channel=self.CHANNEL_ID, oldest=self.LAST_TS, limit=1
- )
- msg = [
- msg
- for msg in resp["messages"]
- if msg.get("user") == get_conf("SLACK_CLAUDE_BOT_ID")
- ]
+ resp = await self.conversations_history(channel=self.CHANNEL_ID, oldest=self.LAST_TS, limit=1)
+ msg = [msg for msg in resp["messages"]
+ if msg.get("user") == get_conf('SLACK_CLAUDE_BOT_ID')[0]]
return msg
except (SlackApiError, KeyError) as e:
raise RuntimeError(f"获取Slack消息失败。")
-
+
async def get_reply(self):
while True:
slack_msgs = await self.get_slack_messages()
if len(slack_msgs) == 0:
await asyncio.sleep(0.5)
continue
-
+
msg = slack_msgs[-1]
if msg["text"].endswith("Typing…_"):
yield False, msg["text"]
else:
yield True, msg["text"]
break
-
except:
pass
"""
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+========================================================================
第二部分：子进程Worker（调用主体）
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+========================================================================
"""
@@ -98,7 +88,7 @@ class ClaudeHandle(Process):
self.success = True
self.local_history = []
self.check_dependency()
- if self.success:
+ if self.success:
self.start()
self.threadLock = threading.Lock()
@@ -106,52 +96,47 @@ class ClaudeHandle(Process):
try:
self.success = False
import slack_sdk
-
self.info = "依赖检测通过，等待Claude响应。注意目前不能多人同时调用Claude接口（有线程锁），否则将导致每个人的Claude问询历史互相渗透。调用Claude时，会自动使用已配置的代理。"
self.success = True
except:
- self.info = "缺少的依赖，如果要使用Claude，除了基础的pip依赖以外，您还需要运行`pip install -r request_llms/requirements_slackclaude.txt`安装Claude的依赖，然后重启程序。"
+ self.info = "缺少的依赖，如果要使用Claude，除了基础的pip依赖以外，您还需要运行`pip install -r request_llm/requirements_slackclaude.txt`安装Claude的依赖，然后重启程序。"
self.success = False
def ready(self):
- return self.claude_model is not None
-
+ return self.claude_model is not None
+
async def async_run(self):
await self.claude_model.open_channel()
while True:
# 等待
kwargs = self.child.recv()
- question = kwargs["query"]
- history = kwargs["history"]
+ question = kwargs['query']
+ history = kwargs['history']
# 开始问问题
prompt = ""
# 问题
prompt += question
- print("question:", prompt)
+ print('question:', prompt)
# 提交
await self.claude_model.chat(prompt)
-
+
# 获取回复
- async for final, response in self.claude_model.get_reply():
+ async for final, response in self.claude_model.get_reply():
if not final:
print(response)
self.child.send(str(response))
else:
# 防止丢失最后一条消息
slack_msgs = await self.claude_model.get_slack_messages()
- last_msg = (
- slack_msgs[-1]["text"]
- if slack_msgs and len(slack_msgs) > 0
- else ""
- )
+ last_msg = slack_msgs[-1]["text"] if slack_msgs and len(slack_msgs) > 0 else ""
if last_msg:
self.child.send(last_msg)
- print("-------- receive final ---------")
- self.child.send("[Finish]")
-
+ print('-------- receive final ---------')
+ self.child.send('[Finish]')
+
def run(self):
"""
这个函数运行在子进程
@@ -161,24 +146,22 @@ class ClaudeHandle(Process):
self.local_history = []
if (self.claude_model is None) or (not self.success):
# 代理设置
- proxies = get_conf("proxies")
+ proxies, = get_conf('proxies')
if proxies is None:
self.proxies_https = None
else:
- self.proxies_https = proxies["https"]
+ self.proxies_https = proxies['https']
try:
- SLACK_CLAUDE_USER_TOKEN = get_conf("SLACK_CLAUDE_USER_TOKEN")
- self.claude_model = SlackClient(
- token=SLACK_CLAUDE_USER_TOKEN, proxy=self.proxies_https
- )
- print("Claude组件初始化成功。")
+ SLACK_CLAUDE_USER_TOKEN, = get_conf('SLACK_CLAUDE_USER_TOKEN')
+ self.claude_model = SlackClient(token=SLACK_CLAUDE_USER_TOKEN, proxy=self.proxies_https)
+ print('Claude组件初始化成功。')
except:
self.success = False
- tb_str = "\n```\n" + trimmed_format_exc() + "\n```\n"
- self.child.send(f"[Local Message] 不能加载Claude组件。{tb_str}")
- self.child.send("[Fail]")
- self.child.send("[Finish]")
+ tb_str = '\n```\n' + trimmed_format_exc() + '\n```\n'
+ self.child.send(f'[Local Message] 不能加载Claude组件。{tb_str}')
+ self.child.send('[Fail]')
+ self.child.send('[Finish]')
raise RuntimeError(f"不能加载Claude组件。")
self.success = True
@@ -186,49 +169,42 @@ class ClaudeHandle(Process):
# 进入任务等待状态
asyncio.run(self.async_run())
except Exception:
- tb_str = "\n```\n" + trimmed_format_exc() + "\n```\n"
- self.child.send(f"[Local Message] Claude失败 {tb_str}.")
- self.child.send("[Fail]")
- self.child.send("[Finish]")
+ tb_str = '\n```\n' + trimmed_format_exc() + '\n```\n'
+ self.child.send(f'[Local Message] Claude失败 {tb_str}.')
+ self.child.send('[Fail]')
+ self.child.send('[Finish]')
def stream_chat(self, **kwargs):
"""
这个函数运行在主进程
"""
self.threadLock.acquire()
- self.parent.send(kwargs) # 发送请求到子进程
+ self.parent.send(kwargs) # 发送请求到子进程
while True:
- res = self.parent.recv() # 等待Claude回复的片段
- if res == "[Finish]":
- break # 结束
- elif res == "[Fail]":
+ res = self.parent.recv() # 等待Claude回复的片段
+ if res == '[Finish]':
+ break # 结束
+ elif res == '[Fail]':
self.success = False
break
else:
- yield res # Claude回复的片段
+ yield res # Claude回复的片段
self.threadLock.release()
"""
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+========================================================================
第三部分：主进程统一调用函数接口
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+========================================================================
"""
global claude_handle
claude_handle = None
-def predict_no_ui_long_connection(
- inputs,
- llm_kwargs,
- history=[],
- sys_prompt="",
- observe_window=None,
- console_slience=False,
-):
+def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="", observe_window=None, console_slience=False):
"""
- 多线程方法
- 函数的说明请见 request_llms/bridge_all.py
+ 多线程方法
+ 函数的说明请见 request_llm/bridge_all.py
"""
global claude_handle
if (claude_handle is None) or (not claude_handle.success):
@@ -241,42 +217,26 @@ def predict_no_ui_long_connection(
# 没有 sys_prompt 接口，因此把prompt加入 history
history_feedin = []
- for i in range(len(history) // 2):
- history_feedin.append([history[2 * i], history[2 * i + 1]])
+ for i in range(len(history)//2):
+ history_feedin.append([history[2*i], history[2*i+1]])
watch_dog_patience = 5 # 看门狗 (watchdog) 的耐心, 设置5秒即可
response = ""
- observe_window[0] = "[Local Message] 等待Claude响应中 ..."
- for response in claude_handle.stream_chat(
- query=inputs,
- history=history_feedin,
- system_prompt=sys_prompt,
- max_length=llm_kwargs["max_length"],
- top_p=llm_kwargs["top_p"],
- temperature=llm_kwargs["temperature"],
- ):
+ observe_window[0] = "[Local Message]: 等待Claude响应中 ..."
+ for response in claude_handle.stream_chat(query=inputs, history=history_feedin, system_prompt=sys_prompt, max_length=llm_kwargs['max_length'], top_p=llm_kwargs['top_p'], temperature=llm_kwargs['temperature']):
observe_window[0] = preprocess_newbing_out_simple(response)
if len(observe_window) >= 2:
- if (time.time() - observe_window[1]) > watch_dog_patience:
+ if (time.time()-observe_window[1]) > watch_dog_patience:
raise RuntimeError("程序终止。")
return preprocess_newbing_out_simple(response)
-def predict(
- inputs,
- llm_kwargs,
- plugin_kwargs,
- chatbot,
- history=[],
- system_prompt="",
- stream=True,
- additional_fn=None,
-):
+def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_prompt='', stream=True, additional_fn=None):
"""
- 单线程方法
- 函数的说明请见 request_llms/bridge_all.py
+ 单线程方法
+ 函数的说明请见 request_llm/bridge_all.py
"""
- chatbot.append((inputs, "[Local Message] 等待Claude响应中 ..."))
+ chatbot.append((inputs, "[Local Message]: 等待Claude响应中 ..."))
global claude_handle
if (claude_handle is None) or (not claude_handle.success):
@@ -289,30 +249,21 @@ def predict(
if additional_fn is not None:
from core_functional import handle_core_functionality
-
- inputs, history = handle_core_functionality(
- additional_fn, inputs, history, chatbot
- )
+ inputs, history = handle_core_functionality(additional_fn, inputs, history, chatbot)
history_feedin = []
- for i in range(len(history) // 2):
- history_feedin.append([history[2 * i], history[2 * i + 1]])
-
- chatbot[-1] = (inputs, "[Local Message] 等待Claude响应中 ...")
- response = "[Local Message] 等待Claude响应中 ..."
- yield from update_ui(
- chatbot=chatbot, history=history, msg="Claude响应缓慢，尚未完成全部响应，请耐心完成后再提交新问题。"
- )
- for response in claude_handle.stream_chat(
- query=inputs, history=history_feedin, system_prompt=system_prompt
- ):
+ for i in range(len(history)//2):
+ history_feedin.append([history[2*i], history[2*i+1]])
+
+ chatbot[-1] = (inputs, "[Local Message]: 等待Claude响应中 ...")
+ response = "[Local Message]: 等待Claude响应中 ..."
+ yield from update_ui(chatbot=chatbot, history=history, msg="Claude响应缓慢，尚未完成全部响应，请耐心完成后再提交新问题。")
+ for response in claude_handle.stream_chat(query=inputs, history=history_feedin, system_prompt=system_prompt):
chatbot[-1] = (inputs, preprocess_newbing_out(response))
- yield from update_ui(
- chatbot=chatbot, history=history, msg="Claude响应缓慢，尚未完成全部响应，请耐心完成后再提交新问题。"
- )
- if response == "[Local Message] 等待Claude响应中 ...":
- response = "[Local Message] Claude响应异常，请刷新界面重试 ..."
+ yield from update_ui(chatbot=chatbot, history=history, msg="Claude响应缓慢，尚未完成全部响应，请耐心完成后再提交新问题。")
+ if response == "[Local Message]: 等待Claude响应中 ...":
+ response = "[Local Message]: Claude响应异常，请刷新界面重试 ..."
history.extend([inputs, response])
- logging.info(f"[raw_input] {inputs}")
- logging.info(f"[response] {response}")
+ logging.info(f'[raw_input] {inputs}')
+ logging.info(f'[response] {response}')
yield from update_ui(chatbot=chatbot, history=history, msg="完成全部响应，请提交新问题。")
diff --git a/request_llms/bridge_tgui.py b/request_llm/bridge_tgui.py
similarity index 100%
rename from request_llms/bridge_tgui.py
rename to request_llm/bridge_tgui.py
diff --git a/request_llms/chatglmoonx.py b/request_llm/chatglmoonx.py
similarity index 100%
rename from request_llms/chatglmoonx.py
rename to request_llm/chatglmoonx.py
diff --git a/request_llms/com_sparkapi.py b/request_llm/com_sparkapi.py
similarity index 78%
rename from request_llms/com_sparkapi.py
rename to request_llm/com_sparkapi.py
index 359e407ae51a945e87acab6f6363fbb4a6507a0a..ae970b9a1cc1d8fb4f87c9f5ee7f558661185428 100644
--- a/request_llms/com_sparkapi.py
+++ b/request_llm/com_sparkapi.py
@@ -1,4 +1,4 @@
-from toolbox import get_conf, get_pictures_list, encode_image
+from toolbox import get_conf
import base64
import datetime
import hashlib
@@ -64,21 +64,18 @@ class SparkRequestInstance():
self.api_key = XFYUN_API_KEY
self.gpt_url = "ws://spark-api.xf-yun.com/v1.1/chat"
self.gpt_url_v2 = "ws://spark-api.xf-yun.com/v2.1/chat"
- self.gpt_url_v3 = "ws://spark-api.xf-yun.com/v3.1/chat"
- self.gpt_url_v35 = "wss://spark-api.xf-yun.com/v3.5/chat"
- self.gpt_url_img = "wss://spark-api.cn-huabei-1.xf-yun.com/v2.1/image"
self.time_to_yield_event = threading.Event()
self.time_to_exit_event = threading.Event()
self.result_buf = ""
- def generate(self, inputs, llm_kwargs, history, system_prompt, use_image_api=False):
+ def generate(self, inputs, llm_kwargs, history, system_prompt):
llm_kwargs = llm_kwargs
history = history
system_prompt = system_prompt
import _thread as thread
- thread.start_new_thread(self.create_blocking_request, (inputs, llm_kwargs, history, system_prompt, use_image_api))
+ thread.start_new_thread(self.create_blocking_request, (inputs, llm_kwargs, history, system_prompt))
while True:
self.time_to_yield_event.wait(timeout=1)
if self.time_to_yield_event.is_set():
@@ -87,22 +84,12 @@ class SparkRequestInstance():
return self.result_buf
- def create_blocking_request(self, inputs, llm_kwargs, history, system_prompt, use_image_api):
+ def create_blocking_request(self, inputs, llm_kwargs, history, system_prompt):
if llm_kwargs['llm_model'] == 'sparkv2':
gpt_url = self.gpt_url_v2
- elif llm_kwargs['llm_model'] == 'sparkv3':
- gpt_url = self.gpt_url_v3
- elif llm_kwargs['llm_model'] == 'sparkv3.5':
- gpt_url = self.gpt_url_v35
else:
gpt_url = self.gpt_url
- file_manifest = []
- if use_image_api and llm_kwargs.get('most_recent_uploaded'):
- if llm_kwargs['most_recent_uploaded'].get('path'):
- file_manifest = get_pictures_list(llm_kwargs['most_recent_uploaded']['path'])
- if len(file_manifest) > 0:
- print('正在使用讯飞图片理解API')
- gpt_url = self.gpt_url_img
+
wsParam = Ws_Param(self.appid, self.api_key, self.api_secret, gpt_url)
websocket.enableTrace(False)
wsUrl = wsParam.create_url()
@@ -111,8 +98,9 @@ class SparkRequestInstance():
def on_open(ws):
import _thread as thread
thread.start_new_thread(run, (ws,))
+
def run(ws, *args):
- data = json.dumps(gen_params(ws.appid, *ws.all_args, file_manifest))
+ data = json.dumps(gen_params(ws.appid, *ws.all_args))
ws.send(data)
# 收到websocket消息的处理
@@ -151,18 +139,9 @@ class SparkRequestInstance():
ws.all_args = (inputs, llm_kwargs, history, system_prompt)
ws.run_forever(sslopt={"cert_reqs": ssl.CERT_NONE})
-def generate_message_payload(inputs, llm_kwargs, history, system_prompt, file_manifest):
+def generate_message_payload(inputs, llm_kwargs, history, system_prompt):
conversation_cnt = len(history) // 2
- messages = []
- if file_manifest:
- base64_images = []
- for image_path in file_manifest:
- base64_images.append(encode_image(image_path))
- for img_s in base64_images:
- if img_s not in str(messages):
- messages.append({"role": "user", "content": img_s, "content_type": "image"})
- else:
- messages = [{"role": "system", "content": system_prompt}]
+ messages = [{"role": "system", "content": system_prompt}]
if conversation_cnt:
for index in range(0, 2*conversation_cnt, 2):
what_i_have_asked = {}
@@ -185,18 +164,10 @@ def generate_message_payload(inputs, llm_kwargs, history, system_prompt, file_ma
return messages
-def gen_params(appid, inputs, llm_kwargs, history, system_prompt, file_manifest):
+def gen_params(appid, inputs, llm_kwargs, history, system_prompt):
"""
通过appid和用户的提问来生成请参数
"""
- domains = {
- "spark": "general",
- "sparkv2": "generalv2",
- "sparkv3": "generalv3",
- "sparkv3.5": "generalv3.5",
- }
- domains_select = domains[llm_kwargs['llm_model']]
- if file_manifest: domains_select = 'image'
data = {
"header": {
"app_id": appid,
@@ -204,7 +175,7 @@ def gen_params(appid, inputs, llm_kwargs, history, system_prompt, file_manifest)
},
"parameter": {
"chat": {
- "domain": domains_select,
+ "domain": "generalv2" if llm_kwargs['llm_model'] == 'sparkv2' else "general",
"temperature": llm_kwargs["temperature"],
"random_threshold": 0.5,
"max_tokens": 4096,
@@ -213,7 +184,7 @@ def gen_params(appid, inputs, llm_kwargs, history, system_prompt, file_manifest)
},
"payload": {
"message": {
- "text": generate_message_payload(inputs, llm_kwargs, history, system_prompt, file_manifest)
+ "text": generate_message_payload(inputs, llm_kwargs, history, system_prompt)
}
}
}
diff --git a/request_llm/edge_gpt.py b/request_llm/edge_gpt.py
new file mode 100644
index 0000000000000000000000000000000000000000..bbf84000d84a42de80d3c051a24f06336af76aaf
--- /dev/null
+++ b/request_llm/edge_gpt.py
@@ -0,0 +1,409 @@
+"""
+========================================================================
+第一部分：来自EdgeGPT.py
+https://github.com/acheong08/EdgeGPT
+========================================================================
+"""
+
+import argparse
+import asyncio
+import json
+import os
+import random
+import re
+import ssl
+import sys
+import uuid
+from enum import Enum
+from typing import Generator
+from typing import Literal
+from typing import Optional
+from typing import Union
+import websockets.client as websockets
+
+DELIMITER = "\x1e"
+
+
+# Generate random IP between range 13.104.0.0/14
+FORWARDED_IP = (
+ f"13.{random.randint(104, 107)}.{random.randint(0, 255)}.{random.randint(0, 255)}"
+)
+
+HEADERS = {
+ "accept": "application/json",
+ "accept-language": "en-US,en;q=0.9",
+ "content-type": "application/json",
+ "sec-ch-ua": '"Not_A Brand";v="99", "Microsoft Edge";v="110", "Chromium";v="110"',
+ "sec-ch-ua-arch": '"x86"',
+ "sec-ch-ua-bitness": '"64"',
+ "sec-ch-ua-full-version": '"109.0.1518.78"',
+ "sec-ch-ua-full-version-list": '"Chromium";v="110.0.5481.192", "Not A(Brand";v="24.0.0.0", "Microsoft Edge";v="110.0.1587.69"',
+ "sec-ch-ua-mobile": "?0",
+ "sec-ch-ua-model": "",
+ "sec-ch-ua-platform": '"Windows"',
+ "sec-ch-ua-platform-version": '"15.0.0"',
+ "sec-fetch-dest": "empty",
+ "sec-fetch-mode": "cors",
+ "sec-fetch-site": "same-origin",
+ "x-ms-client-request-id": str(uuid.uuid4()),
+ "x-ms-useragent": "azsdk-js-api-client-factory/1.0.0-beta.1 core-rest-pipeline/1.10.0 OS/Win32",
+ "Referer": "https://www.bing.com/search?q=Bing+AI&showconv=1&FORM=hpcodx",
+ "Referrer-Policy": "origin-when-cross-origin",
+ "x-forwarded-for": FORWARDED_IP,
+}
+
+HEADERS_INIT_CONVER = {
+ "authority": "edgeservices.bing.com",
+ "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+ "accept-language": "en-US,en;q=0.9",
+ "cache-control": "max-age=0",
+ "sec-ch-ua": '"Chromium";v="110", "Not A(Brand";v="24", "Microsoft Edge";v="110"',
+ "sec-ch-ua-arch": '"x86"',
+ "sec-ch-ua-bitness": '"64"',
+ "sec-ch-ua-full-version": '"110.0.1587.69"',
+ "sec-ch-ua-full-version-list": '"Chromium";v="110.0.5481.192", "Not A(Brand";v="24.0.0.0", "Microsoft Edge";v="110.0.1587.69"',
+ "sec-ch-ua-mobile": "?0",
+ "sec-ch-ua-model": '""',
+ "sec-ch-ua-platform": '"Windows"',
+ "sec-ch-ua-platform-version": '"15.0.0"',
+ "sec-fetch-dest": "document",
+ "sec-fetch-mode": "navigate",
+ "sec-fetch-site": "none",
+ "sec-fetch-user": "?1",
+ "upgrade-insecure-requests": "1",
+ "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 Edg/110.0.1587.69",
+ "x-edge-shopping-flag": "1",
+ "x-forwarded-for": FORWARDED_IP,
+}
+
+def get_ssl_context():
+ import certifi
+ ssl_context = ssl.create_default_context()
+ ssl_context.load_verify_locations(certifi.where())
+ return ssl_context
+
+
+
+class NotAllowedToAccess(Exception):
+ pass
+
+
+class ConversationStyle(Enum):
+ creative = "h3imaginative,clgalileo,gencontentv3"
+ balanced = "galileo"
+ precise = "h3precise,clgalileo"
+
+
+CONVERSATION_STYLE_TYPE = Optional[
+ Union[ConversationStyle, Literal["creative", "balanced", "precise"]]
+]
+
+
+def _append_identifier(msg: dict) -> str:
+ """
+ Appends special character to end of message to identify end of message
+ """
+ # Convert dict to json string
+ return json.dumps(msg) + DELIMITER
+
+
+def _get_ran_hex(length: int = 32) -> str:
+ """
+ Returns random hex string
+ """
+ return "".join(random.choice("0123456789abcdef") for _ in range(length))
+
+
+class _ChatHubRequest:
+ """
+ Request object for ChatHub
+ """
+
+ def __init__(
+ self,
+ conversation_signature: str,
+ client_id: str,
+ conversation_id: str,
+ invocation_id: int = 0,
+ ) -> None:
+ self.struct: dict = {}
+
+ self.client_id: str = client_id
+ self.conversation_id: str = conversation_id
+ self.conversation_signature: str = conversation_signature
+ self.invocation_id: int = invocation_id
+
+ def update(
+ self,
+ prompt,
+ conversation_style,
+ options,
+ ) -> None:
+ """
+ Updates request object
+ """
+ if options is None:
+ options = [
+ "deepleo",
+ "enable_debug_commands",
+ "disable_emoji_spoken_text",
+ "enablemm",
+ ]
+ if conversation_style:
+ if not isinstance(conversation_style, ConversationStyle):
+ conversation_style = getattr(ConversationStyle, conversation_style)
+ options = [
+ "nlu_direct_response_filter",
+ "deepleo",
+ "disable_emoji_spoken_text",
+ "responsible_ai_policy_235",
+ "enablemm",
+ conversation_style.value,
+ "dtappid",
+ "cricinfo",
+ "cricinfov2",
+ "dv3sugg",
+ ]
+ self.struct = {
+ "arguments": [
+ {
+ "source": "cib",
+ "optionsSets": options,
+ "sliceIds": [
+ "222dtappid",
+ "225cricinfo",
+ "224locals0",
+ ],
+ "traceId": _get_ran_hex(32),
+ "isStartOfSession": self.invocation_id == 0,
+ "message": {
+ "author": "user",
+ "inputMethod": "Keyboard",
+ "text": prompt,
+ "messageType": "Chat",
+ },
+ "conversationSignature": self.conversation_signature,
+ "participant": {
+ "id": self.client_id,
+ },
+ "conversationId": self.conversation_id,
+ },
+ ],
+ "invocationId": str(self.invocation_id),
+ "target": "chat",
+ "type": 4,
+ }
+ self.invocation_id += 1
+
+
+class _Conversation:
+ """
+ Conversation API
+ """
+
+ def __init__(
+ self,
+ cookies,
+ proxy,
+ ) -> None:
+ self.struct: dict = {
+ "conversationId": None,
+ "clientId": None,
+ "conversationSignature": None,
+ "result": {"value": "Success", "message": None},
+ }
+ import httpx
+ self.proxy = proxy
+ proxy = (
+ proxy
+ or os.environ.get("all_proxy")
+ or os.environ.get("ALL_PROXY")
+ or os.environ.get("https_proxy")
+ or os.environ.get("HTTPS_PROXY")
+ or None
+ )
+ if proxy is not None and proxy.startswith("socks5h://"):
+ proxy = "socks5://" + proxy[len("socks5h://") :]
+ self.session = httpx.Client(
+ proxies=proxy,
+ timeout=30,
+ headers=HEADERS_INIT_CONVER,
+ )
+ for cookie in cookies:
+ self.session.cookies.set(cookie["name"], cookie["value"])
+
+ # Send GET request
+ response = self.session.get(
+ url=os.environ.get("BING_PROXY_URL")
+ or "https://edgeservices.bing.com/edgesvc/turing/conversation/create",
+ )
+ if response.status_code != 200:
+ response = self.session.get(
+ "https://edge.churchless.tech/edgesvc/turing/conversation/create",
+ )
+ if response.status_code != 200:
+ print(f"Status code: {response.status_code}")
+ print(response.text)
+ print(response.url)
+ raise Exception("Authentication failed")
+ try:
+ self.struct = response.json()
+ except (json.decoder.JSONDecodeError, NotAllowedToAccess) as exc:
+ raise Exception(
+ "Authentication failed. You have not been accepted into the beta.",
+ ) from exc
+ if self.struct["result"]["value"] == "UnauthorizedRequest":
+ raise NotAllowedToAccess(self.struct["result"]["message"])
+
+
+class _ChatHub:
+ """
+ Chat API
+ """
+
+ def __init__(self, conversation) -> None:
+ self.wss = None
+ self.request: _ChatHubRequest
+ self.loop: bool
+ self.task: asyncio.Task
+ print(conversation.struct)
+ self.request = _ChatHubRequest(
+ conversation_signature=conversation.struct["conversationSignature"],
+ client_id=conversation.struct["clientId"],
+ conversation_id=conversation.struct["conversationId"],
+ )
+
+ async def ask_stream(
+ self,
+ prompt: str,
+ wss_link: str,
+ conversation_style: CONVERSATION_STYLE_TYPE = None,
+ raw: bool = False,
+ options: dict = None,
+ ) -> Generator[str, None, None]:
+ """
+ Ask a question to the bot
+ """
+ if self.wss and not self.wss.closed:
+ await self.wss.close()
+ # Check if websocket is closed
+ self.wss = await websockets.connect(
+ wss_link,
+ extra_headers=HEADERS,
+ max_size=None,
+ ssl=get_ssl_context()
+ )
+ await self._initial_handshake()
+ # Construct a ChatHub request
+ self.request.update(
+ prompt=prompt,
+ conversation_style=conversation_style,
+ options=options,
+ )
+ # Send request
+ await self.wss.send(_append_identifier(self.request.struct))
+ final = False
+ while not final:
+ objects = str(await self.wss.recv()).split(DELIMITER)
+ for obj in objects:
+ if obj is None or not obj:
+ continue
+ response = json.loads(obj)
+ if response.get("type") != 2 and raw:
+ yield False, response
+ elif response.get("type") == 1 and response["arguments"][0].get(
+ "messages",
+ ):
+ resp_txt = response["arguments"][0]["messages"][0]["adaptiveCards"][
+ 0
+ ]["body"][0].get("text")
+ yield False, resp_txt
+ elif response.get("type") == 2:
+ final = True
+ yield True, response
+
+ async def _initial_handshake(self) -> None:
+ await self.wss.send(_append_identifier({"protocol": "json", "version": 1}))
+ await self.wss.recv()
+
+ async def close(self) -> None:
+ """
+ Close the connection
+ """
+ if self.wss and not self.wss.closed:
+ await self.wss.close()
+
+
+class NewbingChatbot:
+ """
+ Combines everything to make it seamless
+ """
+
+ def __init__(
+ self,
+ cookies,
+ proxy
+ ) -> None:
+ if cookies is None:
+ cookies = {}
+ self.cookies = cookies
+ self.proxy = proxy
+ self.chat_hub: _ChatHub = _ChatHub(
+ _Conversation(self.cookies, self.proxy),
+ )
+
+ async def ask(
+ self,
+ prompt: str,
+ wss_link: str,
+ conversation_style: CONVERSATION_STYLE_TYPE = None,
+ options: dict = None,
+ ) -> dict:
+ """
+ Ask a question to the bot
+ """
+ async for final, response in self.chat_hub.ask_stream(
+ prompt=prompt,
+ conversation_style=conversation_style,
+ wss_link=wss_link,
+ options=options,
+ ):
+ if final:
+ return response
+ await self.chat_hub.wss.close()
+ return None
+
+ async def ask_stream(
+ self,
+ prompt: str,
+ wss_link: str,
+ conversation_style: CONVERSATION_STYLE_TYPE = None,
+ raw: bool = False,
+ options: dict = None,
+ ) -> Generator[str, None, None]:
+ """
+ Ask a question to the bot
+ """
+ async for response in self.chat_hub.ask_stream(
+ prompt=prompt,
+ conversation_style=conversation_style,
+ wss_link=wss_link,
+ raw=raw,
+ options=options,
+ ):
+ yield response
+
+ async def close(self) -> None:
+ """
+ Close the connection
+ """
+ await self.chat_hub.close()
+
+ async def reset(self) -> None:
+ """
+ Reset the conversation
+ """
+ await self.close()
+ self.chat_hub = _ChatHub(_Conversation(self.cookies, self.proxy))
+
+
diff --git a/request_llms/edge_gpt_free.py b/request_llm/edge_gpt_free.py
similarity index 98%
rename from request_llms/edge_gpt_free.py
rename to request_llm/edge_gpt_free.py
index d0122a52852fcb0f369d2269410569d8cad7c73b..22ff05272b634e8557ceb83501248cc238074366 100644
--- a/request_llms/edge_gpt_free.py
+++ b/request_llm/edge_gpt_free.py
@@ -1,8 +1,8 @@
"""
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+========================================================================
第一部分：来自EdgeGPT.py
https://github.com/acheong08/EdgeGPT
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+========================================================================
"""
"""
Main.py
@@ -196,9 +196,9 @@ class _ChatHubRequest:
self,
prompt: str,
conversation_style: CONVERSATION_STYLE_TYPE,
- options=None,
- webpage_context=None,
- search_result=False,
+ options = None,
+ webpage_context = None,
+ search_result = False,
) -> None:
"""
Updates request object
@@ -294,9 +294,9 @@ class _Conversation:
def __init__(
self,
- proxy=None,
- async_mode=False,
- cookies=None,
+ proxy = None,
+ async_mode = False,
+ cookies = None,
) -> None:
if async_mode:
return
@@ -350,8 +350,8 @@ class _Conversation:
@staticmethod
async def create(
- proxy=None,
- cookies=None,
+ proxy = None,
+ cookies = None,
):
self = _Conversation(async_mode=True)
self.struct = {
@@ -418,8 +418,8 @@ class _ChatHub:
def __init__(
self,
conversation: _Conversation,
- proxy=None,
- cookies=None,
+ proxy = None,
+ cookies = None,
) -> None:
self.session = None
self.wss = None
@@ -441,7 +441,7 @@ class _ChatHub:
conversation_style: CONVERSATION_STYLE_TYPE = None,
raw: bool = False,
options: dict = None,
- webpage_context=None,
+ webpage_context = None,
search_result: bool = False,
) -> Generator[str, None, None]:
"""
@@ -452,12 +452,10 @@ class _ChatHub:
ws_cookies = []
for cookie in self.cookies:
ws_cookies.append(f"{cookie['name']}={cookie['value']}")
- req_header.update(
- {
- "Cookie": ";".join(ws_cookies),
- }
- )
-
+ req_header.update({
+ 'Cookie': ';'.join(ws_cookies),
+ })
+
timeout = aiohttp.ClientTimeout(total=30)
self.session = aiohttp.ClientSession(timeout=timeout)
@@ -523,9 +521,9 @@ class _ChatHub:
msg = await self.wss.receive()
try:
objects = msg.data.split(DELIMITER)
- except:
+ except :
continue
-
+
for obj in objects:
if obj is None or not obj:
continue
@@ -626,8 +624,8 @@ class Chatbot:
def __init__(
self,
- proxy=None,
- cookies=None,
+ proxy = None,
+ cookies = None,
) -> None:
self.proxy = proxy
self.chat_hub: _ChatHub = _ChatHub(
@@ -638,8 +636,8 @@ class Chatbot:
@staticmethod
async def create(
- proxy=None,
- cookies=None,
+ proxy = None,
+ cookies = None,
):
self = Chatbot.__new__(Chatbot)
self.proxy = proxy
@@ -656,7 +654,7 @@ class Chatbot:
wss_link: str = "wss://sydney.bing.com/sydney/ChatHub",
conversation_style: CONVERSATION_STYLE_TYPE = None,
options: dict = None,
- webpage_context=None,
+ webpage_context = None,
search_result: bool = False,
) -> dict:
"""
@@ -682,7 +680,7 @@ class Chatbot:
conversation_style: CONVERSATION_STYLE_TYPE = None,
raw: bool = False,
options: dict = None,
- webpage_context=None,
+ webpage_context = None,
search_result: bool = False,
) -> Generator[str, None, None]:
"""
diff --git a/request_llm/local_llm_class.py b/request_llm/local_llm_class.py
new file mode 100644
index 0000000000000000000000000000000000000000..c9c7253440e9630ec77524598ad9a4a318190739
--- /dev/null
+++ b/request_llm/local_llm_class.py
@@ -0,0 +1,180 @@
+from transformers import AutoModel, AutoTokenizer
+import time
+import threading
+import importlib
+from toolbox import update_ui, get_conf, Singleton
+from multiprocessing import Process, Pipe
+
+def SingletonLocalLLM(cls):
+ """
+ 一个单实例装饰器
+ """
+ _instance = {}
+ def _singleton(*args, **kargs):
+ if cls not in _instance:
+ _instance[cls] = cls(*args, **kargs)
+ return _instance[cls]
+ elif _instance[cls].corrupted:
+ _instance[cls] = cls(*args, **kargs)
+ return _instance[cls]
+ else:
+ return _instance[cls]
+ return _singleton
+
+class LocalLLMHandle(Process):
+ def __init__(self):
+ # ⭐主进程执行
+ super().__init__(daemon=True)
+ self.corrupted = False
+ self.load_model_info()
+ self.parent, self.child = Pipe()
+ self.running = True
+ self._model = None
+ self._tokenizer = None
+ self.info = ""
+ self.check_dependency()
+ self.start()
+ self.threadLock = threading.Lock()
+
+ def load_model_info(self):
+ # 🏃♂️🏃♂️🏃♂️ 子进程执行
+ raise NotImplementedError("Method not implemented yet")
+ self.model_name = ""
+ self.cmd_to_install = ""
+
+ def load_model_and_tokenizer(self):
+ """
+ This function should return the model and the tokenizer
+ """
+ # 🏃♂️🏃♂️🏃♂️ 子进程执行
+ raise NotImplementedError("Method not implemented yet")
+
+ def llm_stream_generator(self, **kwargs):
+ # 🏃♂️🏃♂️🏃♂️ 子进程执行
+ raise NotImplementedError("Method not implemented yet")
+
+ def try_to_import_special_deps(self, **kwargs):
+ """
+ import something that will raise error if the user does not install requirement_*.txt
+ """
+ # ⭐主进程执行
+ raise NotImplementedError("Method not implemented yet")
+
+ def check_dependency(self):
+ # ⭐主进程执行
+ try:
+ self.try_to_import_special_deps()
+ self.info = "依赖检测通过"
+ self.running = True
+ except:
+ self.info = f"缺少{self.model_name}的依赖，如果要使用{self.model_name}，除了基础的pip依赖以外，您还需要运行{self.cmd_to_install}安装{self.model_name}的依赖。"
+ self.running = False
+
+ def run(self):
+ # 🏃♂️🏃♂️🏃♂️ 子进程执行
+ # 第一次运行，加载参数
+ try:
+ self._model, self._tokenizer = self.load_model_and_tokenizer()
+ except:
+ self.running = False
+ from toolbox import trimmed_format_exc
+ self.child.send(f'[Local Message] 不能正常加载{self.model_name}的参数.' + '\n```\n' + trimmed_format_exc() + '\n```\n')
+ self.child.send('[FinishBad]')
+ raise RuntimeError(f"不能正常加载{self.model_name}的参数！")
+
+ while True:
+ # 进入任务等待状态
+ kwargs = self.child.recv()
+ # 收到消息，开始请求
+ try:
+ for response_full in self.llm_stream_generator(**kwargs):
+ self.child.send(response_full)
+ self.child.send('[Finish]')
+ # 请求处理结束，开始下一个循环
+ except:
+ from toolbox import trimmed_format_exc
+ self.child.send(f'[Local Message] 调用{self.model_name}失败.' + '\n```\n' + trimmed_format_exc() + '\n```\n')
+ self.child.send('[Finish]')
+
+ def stream_chat(self, **kwargs):
+ # ⭐主进程执行
+ self.threadLock.acquire()
+ self.parent.send(kwargs)
+ while True:
+ res = self.parent.recv()
+ if res == '[Finish]':
+ break
+ if res == '[FinishBad]':
+ self.running = False
+ self.corrupted = True
+ break
+ else:
+ yield res
+ self.threadLock.release()
+
+
+
+def get_local_llm_predict_fns(LLMSingletonClass, model_name):
+ load_message = f"{model_name}尚未加载，加载需要一段时间。注意，取决于`config.py`的配置，{model_name}消耗大量的内存（CPU）或显存（GPU），也许会导致低配计算机卡死 ……"
+
+ def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="", observe_window=[], console_slience=False):
+ """
+ ⭐多线程方法
+ 函数的说明请见 request_llm/bridge_all.py
+ """
+ _llm_handle = LLMSingletonClass()
+ if len(observe_window) >= 1: observe_window[0] = load_message + "\n\n" + _llm_handle.info
+ if not _llm_handle.running: raise RuntimeError(_llm_handle.info)
+
+ # chatglm 没有 sys_prompt 接口，因此把prompt加入 history
+ history_feedin = []
+ history_feedin.append([sys_prompt, "Certainly!"])
+ for i in range(len(history)//2):
+ history_feedin.append([history[2*i], history[2*i+1]] )
+
+ watch_dog_patience = 5 # 看门狗 (watchdog) 的耐心, 设置5秒即可
+ response = ""
+ for response in _llm_handle.stream_chat(query=inputs, history=history_feedin, max_length=llm_kwargs['max_length'], top_p=llm_kwargs['top_p'], temperature=llm_kwargs['temperature']):
+ if len(observe_window) >= 1:
+ observe_window[0] = response
+ if len(observe_window) >= 2:
+ if (time.time()-observe_window[1]) > watch_dog_patience: raise RuntimeError("程序终止。")
+ return response
+
+
+
+ def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_prompt='', stream = True, additional_fn=None):
+ """
+ ⭐单线程方法
+ 函数的说明请见 request_llm/bridge_all.py
+ """
+ chatbot.append((inputs, ""))
+
+ _llm_handle = LLMSingletonClass()
+ chatbot[-1] = (inputs, load_message + "\n\n" + _llm_handle.info)
+ yield from update_ui(chatbot=chatbot, history=[])
+ if not _llm_handle.running: raise RuntimeError(_llm_handle.info)
+
+ if additional_fn is not None:
+ from core_functional import handle_core_functionality
+ inputs, history = handle_core_functionality(additional_fn, inputs, history, chatbot)
+
+ # 处理历史信息
+ history_feedin = []
+ history_feedin.append([system_prompt, "Certainly!"])
+ for i in range(len(history)//2):
+ history_feedin.append([history[2*i], history[2*i+1]] )
+
+ # 开始接收回复
+ response = f"[Local Message]: 等待{model_name}响应中 ..."
+ for response in _llm_handle.stream_chat(query=inputs, history=history_feedin, max_length=llm_kwargs['max_length'], top_p=llm_kwargs['top_p'], temperature=llm_kwargs['temperature']):
+ chatbot[-1] = (inputs, response)
+ yield from update_ui(chatbot=chatbot, history=history)
+
+ # 总结输出
+ if response == f"[Local Message]: 等待{model_name}响应中 ...":
+ response = f"[Local Message]: {model_name}响应异常 ..."
+ history.extend([inputs, response])
+ yield from update_ui(chatbot=chatbot, history=history)
+
+ return predict_no_ui_long_connection, predict
\ No newline at end of file
diff --git a/request_llms/requirements_chatglm.txt b/request_llm/requirements_chatglm.txt
similarity index 56%
rename from request_llms/requirements_chatglm.txt
rename to request_llm/requirements_chatglm.txt
index dabdd8ea519c3f9e59031bb6b73d55502c89f99a..b2629f83d9dc3e54589d59c02de585afa58772b4 100644
--- a/request_llms/requirements_chatglm.txt
+++ b/request_llm/requirements_chatglm.txt
@@ -1,5 +1,6 @@
protobuf
+transformers>=4.27.1
cpm_kernels
torch>=1.10
mdtex2html
-sentencepiece
+sentencepiece
\ No newline at end of file
diff --git a/request_llms/requirements_chatglm_onnx.txt b/request_llm/requirements_chatglm_onnx.txt
similarity index 66%
rename from request_llms/requirements_chatglm_onnx.txt
rename to request_llm/requirements_chatglm_onnx.txt
index 2cd11f69d06adea5a276d6832e6571b7e66c2a5c..70ab6684a7c3472daa381154f276725ca91840fa 100644
--- a/request_llms/requirements_chatglm_onnx.txt
+++ b/request_llm/requirements_chatglm_onnx.txt
@@ -1,4 +1,5 @@
protobuf
+transformers>=4.27.1
cpm_kernels
torch>=1.10
mdtex2html
@@ -6,3 +7,5 @@ sentencepiece
numpy
onnxruntime
sentencepiece
+streamlit
+streamlit-chat
diff --git a/request_llms/requirements_jittorllms.txt b/request_llm/requirements_jittorllms.txt
similarity index 69%
rename from request_llms/requirements_jittorllms.txt
rename to request_llm/requirements_jittorllms.txt
index 2bd2da8bb2b8cc3624758c52e1bd429d95df607c..1d86ff81eb096542ef8e1a903947ea40f826238a 100644
--- a/request_llms/requirements_jittorllms.txt
+++ b/request_llm/requirements_jittorllms.txt
@@ -2,5 +2,6 @@ jittor >= 1.3.7.9
jtorch >= 0.1.3
torch
torchvision
+transformers==4.26.1
pandas
-jieba
+jieba
\ No newline at end of file
diff --git a/request_llms/requirements_moss.txt b/request_llm/requirements_moss.txt
similarity index 69%
rename from request_llms/requirements_moss.txt
rename to request_llm/requirements_moss.txt
index a1f84f0415d98a848a412d6bce77d90792de633f..8dd75bffbe224648b6bc2c3a3b21bb49803ba8da 100644
--- a/request_llms/requirements_moss.txt
+++ b/request_llm/requirements_moss.txt
@@ -1,7 +1,10 @@
torch
+transformers==4.25.1
sentencepiece
datasets
accelerate
matplotlib
huggingface_hub
triton
+streamlit
+
diff --git a/request_llms/requirements_newbing.txt b/request_llm/requirements_newbing.txt
similarity index 100%
rename from request_llms/requirements_newbing.txt
rename to request_llm/requirements_newbing.txt
diff --git a/request_llm/requirements_qwen.txt b/request_llm/requirements_qwen.txt
new file mode 100644
index 0000000000000000000000000000000000000000..3d7d62a05d2023abc94dbbe1c61ff49fc7796c41
--- /dev/null
+++ b/request_llm/requirements_qwen.txt
@@ -0,0 +1,2 @@
+modelscope
+transformers_stream_generator
\ No newline at end of file
diff --git a/request_llm/requirements_slackclaude.txt b/request_llm/requirements_slackclaude.txt
new file mode 100644
index 0000000000000000000000000000000000000000..472d58c2844b20e171720c873d794959c24fdce7
--- /dev/null
+++ b/request_llm/requirements_slackclaude.txt
@@ -0,0 +1 @@
+slack-sdk==3.21.3
\ No newline at end of file
diff --git a/request_llm/test_llms.py b/request_llm/test_llms.py
new file mode 100644
index 0000000000000000000000000000000000000000..ae6967be7b0c48d4c2af7a51335bd9becbc24d88
--- /dev/null
+++ b/request_llm/test_llms.py
@@ -0,0 +1,78 @@
+# """
+# 对各个llm模型进行单元测试
+# """
+def validate_path():
+ import os, sys
+ dir_name = os.path.dirname(__file__)
+ root_dir_assume = os.path.abspath(os.path.dirname(__file__) + '/..')
+ os.chdir(root_dir_assume)
+ sys.path.append(root_dir_assume)
+
+validate_path() # validate path so you can run from base directory
+if __name__ == "__main__":
+ from request_llm.bridge_newbingfree import predict_no_ui_long_connection
+ # from request_llm.bridge_moss import predict_no_ui_long_connection
+ # from request_llm.bridge_jittorllms_pangualpha import predict_no_ui_long_connection
+ # from request_llm.bridge_jittorllms_llama import predict_no_ui_long_connection
+
+ llm_kwargs = {
+ 'max_length': 512,
+ 'top_p': 1,
+ 'temperature': 1,
+ }
+
+ result = predict_no_ui_long_connection(inputs="你好",
+ llm_kwargs=llm_kwargs,
+ history=[],
+ sys_prompt="")
+ print('final result:', result)
+
+
+ result = predict_no_ui_long_connection(inputs="what is a hero?",
+ llm_kwargs=llm_kwargs,
+ history=["hello world"],
+ sys_prompt="")
+ print('final result:', result)
+
+ result = predict_no_ui_long_connection(inputs="如何理解传奇?",
+ llm_kwargs=llm_kwargs,
+ history=[],
+ sys_prompt="")
+ print('final result:', result)
+
+ # # print(result)
+ # from multiprocessing import Process, Pipe
+ # class GetGLMHandle(Process):
+ # def __init__(self):
+ # super().__init__(daemon=True)
+ # pass
+ # def run(self):
+ # # 子进程执行
+ # # 第一次运行，加载参数
+ # def validate_path():
+ # import os, sys
+ # dir_name = os.path.dirname(__file__)
+ # root_dir_assume = os.path.abspath(os.path.dirname(__file__) + '/..')
+ # os.chdir(root_dir_assume + '/request_llm/jittorllms')
+ # sys.path.append(root_dir_assume + '/request_llm/jittorllms')
+ # validate_path() # validate path so you can run from base directory
+
+ # jittorllms_model = None
+ # import types
+ # try:
+ # if jittorllms_model is None:
+ # from models import get_model
+ # # availabel_models = ["chatglm", "pangualpha", "llama", "chatrwkv"]
+ # args_dict = {'model': 'chatrwkv'}
+ # print('self.jittorllms_model = get_model(types.SimpleNamespace(**args_dict))')
+ # jittorllms_model = get_model(types.SimpleNamespace(**args_dict))
+ # print('done get model')
+ # except:
+ # # self.child.send('[Local Message] Call jittorllms fail 不能正常加载jittorllms的参数。')
+ # raise RuntimeError("不能正常加载jittorllms的参数！")
+
+ # x = GetGLMHandle()
+ # x.start()
+
+
+ # input()
\ No newline at end of file
diff --git a/request_llms/README.md b/request_llms/README.md
deleted file mode 100644
index 5a51592ab8751bed5583532c8d552ba492c54c86..0000000000000000000000000000000000000000
--- a/request_llms/README.md
+++ /dev/null
@@ -1,35 +0,0 @@
-P.S. 如果您按照以下步骤成功接入了新的大模型，欢迎发Pull Requests（如果您在自己接入新模型的过程中遇到困难，欢迎加README底部QQ群联系群主）
-
-
-# 如何接入其他本地大语言模型
-
-1. 复制`request_llms/bridge_llama2.py`，重命名为你喜欢的名字
-
-2. 修改`load_model_and_tokenizer`方法，加载你的模型和分词器（去该模型官网找demo，复制粘贴即可）
-
-3. 修改`llm_stream_generator`方法，定义推理模型（去该模型官网找demo，复制粘贴即可）
-
-4. 命令行测试
- - 修改`tests/test_llms.py`（聪慧如您，只需要看一眼该文件就明白怎么修改了）
- - 运行`python tests/test_llms.py`
-
-5. 测试通过后，在`request_llms/bridge_all.py`中做最后的修改，把你的模型完全接入到框架中（聪慧如您，只需要看一眼该文件就明白怎么修改了）
-
-6. 修改`LLM_MODEL`配置，然后运行`python main.py`，测试最后的效果
-
-
-# 如何接入其他在线大语言模型
-
-1. 复制`request_llms/bridge_zhipu.py`，重命名为你喜欢的名字
-
-2. 修改`predict_no_ui_long_connection`
-
-3. 修改`predict`
-
-4. 命令行测试
- - 修改`tests/test_llms.py`（聪慧如您，只需要看一眼该文件就明白怎么修改了）
- - 运行`python tests/test_llms.py`
-
-5. 测试通过后，在`request_llms/bridge_all.py`中做最后的修改，把你的模型完全接入到框架中（聪慧如您，只需要看一眼该文件就明白怎么修改了）
-
-6. 修改`LLM_MODEL`配置，然后运行`python main.py`，测试最后的效果
diff --git a/request_llms/bridge_chatglm.py b/request_llms/bridge_chatglm.py
deleted file mode 100644
index c58495dccfc7d64f194b5f6904b6660141c41cad..0000000000000000000000000000000000000000
--- a/request_llms/bridge_chatglm.py
+++ /dev/null
@@ -1,78 +0,0 @@
-model_name = "ChatGLM"
-cmd_to_install = "`pip install -r request_llms/requirements_chatglm.txt`"
-
-
-from toolbox import get_conf, ProxyNetworkActivate
-from .local_llm_class import LocalLLMHandle, get_local_llm_predict_fns
-
-
-
-# ------------------------------------------------------------------------------------------------------------------------
-# 🔌💻 Local Model
-# ------------------------------------------------------------------------------------------------------------------------
-class GetGLM2Handle(LocalLLMHandle):
-
- def load_model_info(self):
- # 🏃♂️🏃♂️🏃♂️ 子进程执行
- self.model_name = model_name
- self.cmd_to_install = cmd_to_install
-
- def load_model_and_tokenizer(self):
- # 🏃♂️🏃♂️🏃♂️ 子进程执行
- import os, glob
- import os
- import platform
- from transformers import AutoModel, AutoTokenizer
- LOCAL_MODEL_QUANT, device = get_conf('LOCAL_MODEL_QUANT', 'LOCAL_MODEL_DEVICE')
-
- if LOCAL_MODEL_QUANT == "INT4": # INT4
- _model_name_ = "THUDM/chatglm2-6b-int4"
- elif LOCAL_MODEL_QUANT == "INT8": # INT8
- _model_name_ = "THUDM/chatglm2-6b-int8"
- else:
- _model_name_ = "THUDM/chatglm2-6b" # FP16
-
- with ProxyNetworkActivate('Download_LLM'):
- chatglm_tokenizer = AutoTokenizer.from_pretrained(_model_name_, trust_remote_code=True)
- if device=='cpu':
- chatglm_model = AutoModel.from_pretrained(_model_name_, trust_remote_code=True).float()
- else:
- chatglm_model = AutoModel.from_pretrained(_model_name_, trust_remote_code=True).half().cuda()
- chatglm_model = chatglm_model.eval()
-
- self._model = chatglm_model
- self._tokenizer = chatglm_tokenizer
- return self._model, self._tokenizer
-
- def llm_stream_generator(self, **kwargs):
- # 🏃♂️🏃♂️🏃♂️ 子进程执行
- def adaptor(kwargs):
- query = kwargs['query']
- max_length = kwargs['max_length']
- top_p = kwargs['top_p']
- temperature = kwargs['temperature']
- history = kwargs['history']
- return query, max_length, top_p, temperature, history
-
- query, max_length, top_p, temperature, history = adaptor(kwargs)
-
- for response, history in self._model.stream_chat(self._tokenizer,
- query,
- history,
- max_length=max_length,
- top_p=top_p,
- temperature=temperature,
- ):
- yield response
-
- def try_to_import_special_deps(self, **kwargs):
- # import something that will raise error if the user does not install requirement_*.txt
- # 🏃♂️🏃♂️🏃♂️ 主进程执行
- import importlib
- # importlib.import_module('modelscope')
-
-
-# ------------------------------------------------------------------------------------------------------------------------
-# 🔌💻 GPT-Academic Interface
-# ------------------------------------------------------------------------------------------------------------------------
-predict_no_ui_long_connection, predict = get_local_llm_predict_fns(GetGLM2Handle, model_name)
\ No newline at end of file
diff --git a/request_llms/bridge_chatglm3.py b/request_llms/bridge_chatglm3.py
deleted file mode 100644
index 3caa4769d39ae8f56780caf43ddff6373600410d..0000000000000000000000000000000000000000
--- a/request_llms/bridge_chatglm3.py
+++ /dev/null
@@ -1,77 +0,0 @@
-model_name = "ChatGLM3"
-cmd_to_install = "`pip install -r request_llms/requirements_chatglm.txt`"
-
-
-from toolbox import get_conf, ProxyNetworkActivate
-from .local_llm_class import LocalLLMHandle, get_local_llm_predict_fns
-
-
-
-# ------------------------------------------------------------------------------------------------------------------------
-# 🔌💻 Local Model
-# ------------------------------------------------------------------------------------------------------------------------
-class GetGLM3Handle(LocalLLMHandle):
-
- def load_model_info(self):
- # 🏃♂️🏃♂️🏃♂️ 子进程执行
- self.model_name = model_name
- self.cmd_to_install = cmd_to_install
-
- def load_model_and_tokenizer(self):
- # 🏃♂️🏃♂️🏃♂️ 子进程执行
- from transformers import AutoModel, AutoTokenizer
- import os, glob
- import os
- import platform
- LOCAL_MODEL_QUANT, device = get_conf('LOCAL_MODEL_QUANT', 'LOCAL_MODEL_DEVICE')
-
- if LOCAL_MODEL_QUANT == "INT4": # INT4
- _model_name_ = "THUDM/chatglm3-6b-int4"
- elif LOCAL_MODEL_QUANT == "INT8": # INT8
- _model_name_ = "THUDM/chatglm3-6b-int8"
- else:
- _model_name_ = "THUDM/chatglm3-6b" # FP16
- with ProxyNetworkActivate('Download_LLM'):
- chatglm_tokenizer = AutoTokenizer.from_pretrained(_model_name_, trust_remote_code=True)
- if device=='cpu':
- chatglm_model = AutoModel.from_pretrained(_model_name_, trust_remote_code=True, device='cpu').float()
- else:
- chatglm_model = AutoModel.from_pretrained(_model_name_, trust_remote_code=True, device='cuda')
- chatglm_model = chatglm_model.eval()
-
- self._model = chatglm_model
- self._tokenizer = chatglm_tokenizer
- return self._model, self._tokenizer
-
- def llm_stream_generator(self, **kwargs):
- # 🏃♂️🏃♂️🏃♂️ 子进程执行
- def adaptor(kwargs):
- query = kwargs['query']
- max_length = kwargs['max_length']
- top_p = kwargs['top_p']
- temperature = kwargs['temperature']
- history = kwargs['history']
- return query, max_length, top_p, temperature, history
-
- query, max_length, top_p, temperature, history = adaptor(kwargs)
-
- for response, history in self._model.stream_chat(self._tokenizer,
- query,
- history,
- max_length=max_length,
- top_p=top_p,
- temperature=temperature,
- ):
- yield response
-
- def try_to_import_special_deps(self, **kwargs):
- # import something that will raise error if the user does not install requirement_*.txt
- # 🏃♂️🏃♂️🏃♂️ 主进程执行
- import importlib
- # importlib.import_module('modelscope')
-
-
-# ------------------------------------------------------------------------------------------------------------------------
-# 🔌💻 GPT-Academic Interface
-# ------------------------------------------------------------------------------------------------------------------------
-predict_no_ui_long_connection, predict = get_local_llm_predict_fns(GetGLM3Handle, model_name, history_format='chatglm3')
\ No newline at end of file
diff --git a/request_llms/bridge_chatgpt_vision.py b/request_llms/bridge_chatgpt_vision.py
deleted file mode 100644
index ebcf9689a3d74fed75531619bd02dd27993dad5e..0000000000000000000000000000000000000000
--- a/request_llms/bridge_chatgpt_vision.py
+++ /dev/null
@@ -1,312 +0,0 @@
-"""
- 该文件中主要包含三个函数
-
- 不具备多线程能力的函数：
- 1. predict: 正常对话时使用，具备完备的交互功能，不可多线程
-
- 具备多线程调用能力的函数
- 2. predict_no_ui_long_connection：支持多线程
-"""
-
-import json
-import time
-import logging
-import requests
-import base64
-import os
-import glob
-from toolbox import get_conf, update_ui, is_any_api_key, select_api_key, what_keys, clip_history, trimmed_format_exc, is_the_upload_folder, \
- update_ui_lastest_msg, get_max_token, encode_image, have_any_recent_upload_image_files
-
-
-proxies, TIMEOUT_SECONDS, MAX_RETRY, API_ORG, AZURE_CFG_ARRAY = \
- get_conf('proxies', 'TIMEOUT_SECONDS', 'MAX_RETRY', 'API_ORG', 'AZURE_CFG_ARRAY')
-
-timeout_bot_msg = '[Local Message] Request timeout. Network error. Please check proxy settings in config.py.' + \
- '网络错误，检查代理服务器是否可用，以及代理设置的格式是否正确，格式须是[协议]://[地址]:[端口]，缺一不可。'
-
-
-def report_invalid_key(key):
- if get_conf("BLOCK_INVALID_APIKEY"):
- # 实验性功能，自动检测并屏蔽失效的KEY，请勿使用
- from request_llms.key_manager import ApiKeyManager
- api_key = ApiKeyManager().add_key_to_blacklist(key)
-
-def get_full_error(chunk, stream_response):
- """
- 获取完整的从Openai返回的报错
- """
- while True:
- try:
- chunk += next(stream_response)
- except:
- break
- return chunk
-
-def decode_chunk(chunk):
- # 提前读取一些信息 （用于判断异常）
- chunk_decoded = chunk.decode()
- chunkjson = None
- has_choices = False
- choice_valid = False
- has_content = False
- has_role = False
- try:
- chunkjson = json.loads(chunk_decoded[6:])
- has_choices = 'choices' in chunkjson
- if has_choices: choice_valid = (len(chunkjson['choices']) > 0)
- if has_choices and choice_valid: has_content = "content" in chunkjson['choices'][0]["delta"]
- if has_choices and choice_valid: has_role = "role" in chunkjson['choices'][0]["delta"]
- except:
- pass
- return chunk_decoded, chunkjson, has_choices, choice_valid, has_content, has_role
-
-from functools import lru_cache
-@lru_cache(maxsize=32)
-def verify_endpoint(endpoint):
- """
- 检查endpoint是否可用
- """
- return endpoint
-
-def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="", observe_window=None, console_slience=False):
- raise NotImplementedError
-
-
-def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_prompt='', stream = True, additional_fn=None):
-
- have_recent_file, image_paths = have_any_recent_upload_image_files(chatbot)
-
- if is_any_api_key(inputs):
- chatbot._cookies['api_key'] = inputs
- chatbot.append(("输入已识别为openai的api_key", what_keys(inputs)))
- yield from update_ui(chatbot=chatbot, history=history, msg="api_key已导入") # 刷新界面
- return
- elif not is_any_api_key(chatbot._cookies['api_key']):
- chatbot.append((inputs, "缺少api_key。\n\n1. 临时解决方案：直接在输入区键入api_key，然后回车提交。\n\n2. 长效解决方案：在config.py中配置。"))
- yield from update_ui(chatbot=chatbot, history=history, msg="缺少api_key") # 刷新界面
- return
- if not have_recent_file:
- chatbot.append((inputs, "没有检测到任何近期上传的图像文件，请上传jpg格式的图片，此外，请注意拓展名需要小写"))
- yield from update_ui(chatbot=chatbot, history=history, msg="等待图片") # 刷新界面
- return
- if os.path.exists(inputs):
- chatbot.append((inputs, "已经接收到您上传的文件，您不需要再重复强调该文件的路径了，请直接输入您的问题。"))
- yield from update_ui(chatbot=chatbot, history=history, msg="等待指令") # 刷新界面
- return
-
-
- user_input = inputs
- if additional_fn is not None:
- from core_functional import handle_core_functionality
- inputs, history = handle_core_functionality(additional_fn, inputs, history, chatbot)
-
- raw_input = inputs
- logging.info(f'[raw_input] {raw_input}')
- def make_media_input(inputs, image_paths):
- for image_path in image_paths:
- inputs = inputs + f'
'
- return inputs
- chatbot.append((make_media_input(inputs, image_paths), ""))
- yield from update_ui(chatbot=chatbot, history=history, msg="等待响应") # 刷新界面
-
- # check mis-behavior
- if is_the_upload_folder(user_input):
- chatbot[-1] = (inputs, f"[Local Message] 检测到操作错误！当您上传文档之后，需点击“**函数插件区**”按钮进行处理，请勿点击“提交”按钮或者“基础功能区”按钮。")
- yield from update_ui(chatbot=chatbot, history=history, msg="正常") # 刷新界面
- time.sleep(2)
-
- try:
- headers, payload, api_key = generate_payload(inputs, llm_kwargs, history, system_prompt, image_paths)
- except RuntimeError as e:
- chatbot[-1] = (inputs, f"您提供的api-key不满足要求，不包含任何可用于{llm_kwargs['llm_model']}的api-key。您可能选择了错误的模型或请求源。")
- yield from update_ui(chatbot=chatbot, history=history, msg="api-key不满足要求") # 刷新界面
- return
-
- # 检查endpoint是否合法
- try:
- from .bridge_all import model_info
- endpoint = verify_endpoint(model_info[llm_kwargs['llm_model']]['endpoint'])
- except:
- tb_str = '```\n' + trimmed_format_exc() + '```'
- chatbot[-1] = (inputs, tb_str)
- yield from update_ui(chatbot=chatbot, history=history, msg="Endpoint不满足要求") # 刷新界面
- return
-
- history.append(make_media_input(inputs, image_paths))
- history.append("")
-
- retry = 0
- while True:
- try:
- # make a POST request to the API endpoint, stream=True
- response = requests.post(endpoint, headers=headers, proxies=proxies,
- json=payload, stream=True, timeout=TIMEOUT_SECONDS);break
- except:
- retry += 1
- chatbot[-1] = ((chatbot[-1][0], timeout_bot_msg))
- retry_msg = f"，正在重试 ({retry}/{MAX_RETRY}) ……" if MAX_RETRY > 0 else ""
- yield from update_ui(chatbot=chatbot, history=history, msg="请求超时"+retry_msg) # 刷新界面
- if retry > MAX_RETRY: raise TimeoutError
-
- gpt_replying_buffer = ""
-
- is_head_of_the_stream = True
- if stream:
- stream_response = response.iter_lines()
- while True:
- try:
- chunk = next(stream_response)
- except StopIteration:
- # 非OpenAI官方接口的出现这样的报错，OpenAI和API2D不会走这里
- chunk_decoded = chunk.decode()
- error_msg = chunk_decoded
- # 首先排除一个one-api没有done数据包的第三方Bug情形
- if len(gpt_replying_buffer.strip()) > 0 and len(error_msg) == 0:
- yield from update_ui(chatbot=chatbot, history=history, msg="检测到有缺陷的非OpenAI官方接口，建议选择更稳定的接口。")
- break
- # 其他情况，直接返回报错
- chatbot, history = handle_error(inputs, llm_kwargs, chatbot, history, chunk_decoded, error_msg, api_key)
- yield from update_ui(chatbot=chatbot, history=history, msg="非OpenAI官方接口返回了错误:" + chunk.decode()) # 刷新界面
- return
-
- # 提前读取一些信息 （用于判断异常）
- chunk_decoded, chunkjson, has_choices, choice_valid, has_content, has_role = decode_chunk(chunk)
-
- if is_head_of_the_stream and (r'"object":"error"' not in chunk_decoded) and (r"content" not in chunk_decoded):
- # 数据流的第一帧不携带content
- is_head_of_the_stream = False; continue
-
- if chunk:
- try:
- if has_choices and not choice_valid:
- # 一些垃圾第三方接口的出现这样的错误
- continue
- # 前者是API2D的结束条件，后者是OPENAI的结束条件
- if ('data: [DONE]' in chunk_decoded) or (len(chunkjson['choices'][0]["delta"]) == 0):
- # 判定为数据流的结束，gpt_replying_buffer也写完了
- lastmsg = chatbot[-1][-1] + f"\n\n\n\n「{llm_kwargs['llm_model']}调用结束，该模型不具备上下文对话能力，如需追问，请及时切换模型。」"
- yield from update_ui_lastest_msg(lastmsg, chatbot, history, delay=1)
- logging.info(f'[response] {gpt_replying_buffer}')
- break
- # 处理数据流的主体
- status_text = f"finish_reason: {chunkjson['choices'][0].get('finish_reason', 'null')}"
- # 如果这里抛出异常，一般是文本过长，详情见get_full_error的输出
- if has_content:
- # 正常情况
- gpt_replying_buffer = gpt_replying_buffer + chunkjson['choices'][0]["delta"]["content"]
- elif has_role:
- # 一些第三方接口的出现这样的错误，兼容一下吧
- continue
- else:
- # 一些垃圾第三方接口的出现这样的错误
- gpt_replying_buffer = gpt_replying_buffer + chunkjson['choices'][0]["delta"]["content"]
-
- history[-1] = gpt_replying_buffer
- chatbot[-1] = (history[-2], history[-1])
- yield from update_ui(chatbot=chatbot, history=history, msg=status_text) # 刷新界面
- except Exception as e:
- yield from update_ui(chatbot=chatbot, history=history, msg="Json解析不合常规") # 刷新界面
- chunk = get_full_error(chunk, stream_response)
- chunk_decoded = chunk.decode()
- error_msg = chunk_decoded
- chatbot, history = handle_error(inputs, llm_kwargs, chatbot, history, chunk_decoded, error_msg, api_key)
- yield from update_ui(chatbot=chatbot, history=history, msg="Json异常" + error_msg) # 刷新界面
- print(error_msg)
- return
-
-def handle_error(inputs, llm_kwargs, chatbot, history, chunk_decoded, error_msg, api_key=""):
- from .bridge_all import model_info
- openai_website = ' 请登录OpenAI查看详情 https://platform.openai.com/signup'
- if "reduce the length" in error_msg:
- if len(history) >= 2: history[-1] = ""; history[-2] = "" # 清除当前溢出的输入：history[-2] 是本次输入, history[-1] 是本次输出
- history = clip_history(inputs=inputs, history=history, tokenizer=model_info[llm_kwargs['llm_model']]['tokenizer'],
- max_token_limit=(model_info[llm_kwargs['llm_model']]['max_token'])) # history至少释放二分之一
- chatbot[-1] = (chatbot[-1][0], "[Local Message] Reduce the length. 本次输入过长, 或历史数据过长. 历史缓存数据已部分释放, 您可以请再次尝试. (若再次失败则更可能是因为输入过长.)")
- elif "does not exist" in error_msg:
- chatbot[-1] = (chatbot[-1][0], f"[Local Message] Model {llm_kwargs['llm_model']} does not exist. 模型不存在, 或者您没有获得体验资格.")
- elif "Incorrect API key" in error_msg:
- chatbot[-1] = (chatbot[-1][0], "[Local Message] Incorrect API key. OpenAI以提供了不正确的API_KEY为由, 拒绝服务. " + openai_website); report_invalid_key(api_key)
- elif "exceeded your current quota" in error_msg:
- chatbot[-1] = (chatbot[-1][0], "[Local Message] You exceeded your current quota. OpenAI以账户额度不足为由, 拒绝服务." + openai_website); report_invalid_key(api_key)
- elif "account is not active" in error_msg:
- chatbot[-1] = (chatbot[-1][0], "[Local Message] Your account is not active. OpenAI以账户失效为由, 拒绝服务." + openai_website); report_invalid_key(api_key)
- elif "associated with a deactivated account" in error_msg:
- chatbot[-1] = (chatbot[-1][0], "[Local Message] You are associated with a deactivated account. OpenAI以账户失效为由, 拒绝服务." + openai_website); report_invalid_key(api_key)
- elif "API key has been deactivated" in error_msg:
- chatbot[-1] = (chatbot[-1][0], "[Local Message] API key has been deactivated. OpenAI以账户失效为由, 拒绝服务." + openai_website); report_invalid_key(api_key)
- elif "bad forward key" in error_msg:
- chatbot[-1] = (chatbot[-1][0], "[Local Message] Bad forward key. API2D账户额度不足.")
- elif "Not enough point" in error_msg:
- chatbot[-1] = (chatbot[-1][0], "[Local Message] Not enough point. API2D账户点数不足.")
- else:
- from toolbox import regular_txt_to_markdown
- tb_str = '```\n' + trimmed_format_exc() + '```'
- chatbot[-1] = (chatbot[-1][0], f"[Local Message] 异常 \n\n{tb_str} \n\n{regular_txt_to_markdown(chunk_decoded)}")
- return chatbot, history
-
-
-def generate_payload(inputs, llm_kwargs, history, system_prompt, image_paths):
- """
- 整合所有信息，选择LLM模型，生成http请求，为发送请求做准备
- """
- if not is_any_api_key(llm_kwargs['api_key']):
- raise AssertionError("你提供了错误的API_KEY。\n\n1. 临时解决方案：直接在输入区键入api_key，然后回车提交。\n\n2. 长效解决方案：在config.py中配置。")
-
- api_key = select_api_key(llm_kwargs['api_key'], llm_kwargs['llm_model'])
-
- headers = {
- "Content-Type": "application/json",
- "Authorization": f"Bearer {api_key}"
- }
- if API_ORG.startswith('org-'): headers.update({"OpenAI-Organization": API_ORG})
- if llm_kwargs['llm_model'].startswith('azure-'):
- headers.update({"api-key": api_key})
- if llm_kwargs['llm_model'] in AZURE_CFG_ARRAY.keys():
- azure_api_key_unshared = AZURE_CFG_ARRAY[llm_kwargs['llm_model']]["AZURE_API_KEY"]
- headers.update({"api-key": azure_api_key_unshared})
-
- base64_images = []
- for image_path in image_paths:
- base64_images.append(encode_image(image_path))
-
- messages = []
- what_i_ask_now = {}
- what_i_ask_now["role"] = "user"
- what_i_ask_now["content"] = []
- what_i_ask_now["content"].append({
- "type": "text",
- "text": inputs
- })
-
- for image_path, base64_image in zip(image_paths, base64_images):
- what_i_ask_now["content"].append({
- "type": "image_url",
- "image_url": {
- "url": f"data:image/jpeg;base64,{base64_image}"
- }
- })
-
- messages.append(what_i_ask_now)
- model = llm_kwargs['llm_model']
- if llm_kwargs['llm_model'].startswith('api2d-'):
- model = llm_kwargs['llm_model'][len('api2d-'):]
-
- payload = {
- "model": model,
- "messages": messages,
- "temperature": llm_kwargs['temperature'], # 1.0,
- "top_p": llm_kwargs['top_p'], # 1.0,
- "n": 1,
- "stream": True,
- "max_tokens": get_max_token(llm_kwargs),
- "presence_penalty": 0,
- "frequency_penalty": 0,
- }
- try:
- print(f" {llm_kwargs['llm_model']} : {inputs[:100]} ..........")
- except:
- print('输入中可能存在乱码。')
- return headers, payload, api_key
-
-
diff --git a/request_llms/bridge_deepseekcoder.py b/request_llms/bridge_deepseekcoder.py
deleted file mode 100644
index 89964abeea71de3790abfa222177972ea7fc1d75..0000000000000000000000000000000000000000
--- a/request_llms/bridge_deepseekcoder.py
+++ /dev/null
@@ -1,129 +0,0 @@
-model_name = "deepseek-coder-6.7b-instruct"
-cmd_to_install = "未知" # "`pip install -r request_llms/requirements_qwen.txt`"
-
-import os
-from toolbox import ProxyNetworkActivate
-from toolbox import get_conf
-from .local_llm_class import LocalLLMHandle, get_local_llm_predict_fns
-from threading import Thread
-import torch
-
-def download_huggingface_model(model_name, max_retry, local_dir):
- from huggingface_hub import snapshot_download
- for i in range(1, max_retry):
- try:
- snapshot_download(repo_id=model_name, local_dir=local_dir, resume_download=True)
- break
- except Exception as e:
- print(f'\n\n下载失败，重试第{i}次中...\n\n')
- return local_dir
-# ------------------------------------------------------------------------------------------------------------------------
-# 🔌💻 Local Model
-# ------------------------------------------------------------------------------------------------------------------------
-class GetCoderLMHandle(LocalLLMHandle):
-
- def load_model_info(self):
- # 🏃♂️🏃♂️🏃♂️ 子进程执行
- self.model_name = model_name
- self.cmd_to_install = cmd_to_install
-
- def load_model_and_tokenizer(self):
- # 🏃♂️🏃♂️🏃♂️ 子进程执行
- with ProxyNetworkActivate('Download_LLM'):
- from transformers import AutoTokenizer, AutoModelForCausalLM, TextIteratorStreamer
- model_name = "deepseek-ai/deepseek-coder-6.7b-instruct"
- # local_dir = f"~/.cache/{model_name}"
- # if not os.path.exists(local_dir):
- # tokenizer = download_huggingface_model(model_name, max_retry=128, local_dir=local_dir)
- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
- self._streamer = TextIteratorStreamer(tokenizer)
- device_map = {
- "transformer.word_embeddings": 0,
- "transformer.word_embeddings_layernorm": 0,
- "lm_head": 0,
- "transformer.h": 0,
- "transformer.ln_f": 0,
- "model.embed_tokens": 0,
- "model.layers": 0,
- "model.norm": 0,
- }
-
- # 检查量化配置
- quantization_type = get_conf('LOCAL_MODEL_QUANT')
-
- if get_conf('LOCAL_MODEL_DEVICE') != 'cpu':
- if quantization_type == "INT8":
- from transformers import BitsAndBytesConfig
- # 使用 INT8 量化
- model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, load_in_8bit=True,
- device_map=device_map)
- elif quantization_type == "INT4":
- from transformers import BitsAndBytesConfig
- # 使用 INT4 量化
- bnb_config = BitsAndBytesConfig(
- load_in_4bit=True,
- bnb_4bit_use_double_quant=True,
- bnb_4bit_quant_type="nf4",
- bnb_4bit_compute_dtype=torch.bfloat16
- )
- model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True,
- quantization_config=bnb_config, device_map=device_map)
- else:
- # 使用默认的 FP16
- model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True,
- torch_dtype=torch.bfloat16, device_map=device_map)
- else:
- # CPU 模式
- model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True,
- torch_dtype=torch.bfloat16)
-
- return model, tokenizer
-
- def llm_stream_generator(self, **kwargs):
- # 🏃♂️🏃♂️🏃♂️ 子进程执行
- def adaptor(kwargs):
- query = kwargs['query']
- max_length = kwargs['max_length']
- top_p = kwargs['top_p']
- temperature = kwargs['temperature']
- history = kwargs['history']
- return query, max_length, top_p, temperature, history
-
- query, max_length, top_p, temperature, history = adaptor(kwargs)
- history.append({ 'role': 'user', 'content': query})
- messages = history
- inputs = self._tokenizer.apply_chat_template(messages, return_tensors="pt")
- if inputs.shape[1] > max_length:
- inputs = inputs[:, -max_length:]
- inputs = inputs.to(self._model.device)
- generation_kwargs = dict(
- inputs=inputs,
- max_new_tokens=max_length,
- do_sample=False,
- top_p=top_p,
- streamer = self._streamer,
- top_k=50,
- temperature=temperature,
- num_return_sequences=1,
- eos_token_id=32021,
- )
- thread = Thread(target=self._model.generate, kwargs=generation_kwargs, daemon=True)
- thread.start()
- generated_text = ""
- for new_text in self._streamer:
- generated_text += new_text
- # print(generated_text)
- yield generated_text
-
-
- def try_to_import_special_deps(self, **kwargs): pass
- # import something that will raise error if the user does not install requirement_*.txt
- # 🏃♂️🏃♂️🏃♂️ 主进程执行
- # import importlib
- # importlib.import_module('modelscope')
-
-
-# ------------------------------------------------------------------------------------------------------------------------
-# 🔌💻 GPT-Academic Interface
-# ------------------------------------------------------------------------------------------------------------------------
-predict_no_ui_long_connection, predict = get_local_llm_predict_fns(GetCoderLMHandle, model_name, history_format='chatglm3')
\ No newline at end of file
diff --git a/request_llms/bridge_google_gemini.py b/request_llms/bridge_google_gemini.py
deleted file mode 100644
index cb85ecb6d342481d2036695dfab8b2a63cd8c70a..0000000000000000000000000000000000000000
--- a/request_llms/bridge_google_gemini.py
+++ /dev/null
@@ -1,118 +0,0 @@
-# encoding: utf-8
-# @Time : 2023/12/21
-# @Author : Spike
-# @Descr :
-import json
-import re
-import os
-import time
-from request_llms.com_google import GoogleChatInit
-from toolbox import get_conf, update_ui, update_ui_lastest_msg, have_any_recent_upload_image_files, trimmed_format_exc
-
-proxies, TIMEOUT_SECONDS, MAX_RETRY = get_conf('proxies', 'TIMEOUT_SECONDS', 'MAX_RETRY')
-timeout_bot_msg = '[Local Message] Request timeout. Network error. Please check proxy settings in config.py.' + \
- '网络错误，检查代理服务器是否可用，以及代理设置的格式是否正确，格式须是[协议]://[地址]:[端口]，缺一不可。'
-
-
-def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="", observe_window=None,
- console_slience=False):
- # 检查API_KEY
- if get_conf("GEMINI_API_KEY") == "":
- raise ValueError(f"请配置 GEMINI_API_KEY。")
-
- genai = GoogleChatInit()
- watch_dog_patience = 5 # 看门狗的耐心, 设置5秒即可
- gpt_replying_buffer = ''
- stream_response = genai.generate_chat(inputs, llm_kwargs, history, sys_prompt)
- for response in stream_response:
- results = response.decode()
- match = re.search(r'"text":\s*"((?:[^"\\]|\\.)*)"', results, flags=re.DOTALL)
- error_match = re.search(r'\"message\":\s*\"(.*?)\"', results, flags=re.DOTALL)
- if match:
- try:
- paraphrase = json.loads('{"text": "%s"}' % match.group(1))
- except:
- raise ValueError(f"解析GEMINI消息出错。")
- buffer = paraphrase['text']
- gpt_replying_buffer += buffer
- if len(observe_window) >= 1:
- observe_window[0] = gpt_replying_buffer
- if len(observe_window) >= 2:
- if (time.time() - observe_window[1]) > watch_dog_patience: raise RuntimeError("程序终止。")
- if error_match:
- raise RuntimeError(f'{gpt_replying_buffer} 对话错误')
- return gpt_replying_buffer
-
-
-def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_prompt='', stream=True, additional_fn=None):
- # 检查API_KEY
- if get_conf("GEMINI_API_KEY") == "":
- yield from update_ui_lastest_msg(f"请配置 GEMINI_API_KEY。", chatbot=chatbot, history=history, delay=0)
- return
-
- # 适配润色区域
- if additional_fn is not None:
- from core_functional import handle_core_functionality
- inputs, history = handle_core_functionality(additional_fn, inputs, history, chatbot)
-
- if "vision" in llm_kwargs["llm_model"]:
- have_recent_file, image_paths = have_any_recent_upload_image_files(chatbot)
- if not have_recent_file:
- chatbot.append((inputs, "没有检测到任何近期上传的图像文件，请上传jpg格式的图片，此外，请注意拓展名需要小写"))
- yield from update_ui(chatbot=chatbot, history=history, msg="等待图片") # 刷新界面
- return
- def make_media_input(inputs, image_paths):
- for image_path in image_paths:
- inputs = inputs + f'
'
- return inputs
- if have_recent_file:
- inputs = make_media_input(inputs, image_paths)
-
- chatbot.append((inputs, ""))
- yield from update_ui(chatbot=chatbot, history=history)
- genai = GoogleChatInit()
- retry = 0
- while True:
- try:
- stream_response = genai.generate_chat(inputs, llm_kwargs, history, system_prompt)
- break
- except Exception as e:
- retry += 1
- chatbot[-1] = ((chatbot[-1][0], trimmed_format_exc()))
- yield from update_ui(chatbot=chatbot, history=history, msg="请求失败") # 刷新界面
- return
- gpt_replying_buffer = ""
- gpt_security_policy = ""
- history.extend([inputs, ''])
- for response in stream_response:
- results = response.decode("utf-8") # 被这个解码给耍了。。
- gpt_security_policy += results
- match = re.search(r'"text":\s*"((?:[^"\\]|\\.)*)"', results, flags=re.DOTALL)
- error_match = re.search(r'\"message\":\s*\"(.*)\"', results, flags=re.DOTALL)
- if match:
- try:
- paraphrase = json.loads('{"text": "%s"}' % match.group(1))
- except:
- raise ValueError(f"解析GEMINI消息出错。")
- gpt_replying_buffer += paraphrase['text'] # 使用 json 解析库进行处理
- chatbot[-1] = (inputs, gpt_replying_buffer)
- history[-1] = gpt_replying_buffer
- yield from update_ui(chatbot=chatbot, history=history)
- if error_match:
- history = history[-2] # 错误的不纳入对话
- chatbot[-1] = (inputs, gpt_replying_buffer + f"对话错误，请查看message\n\n```\n{error_match.group(1)}\n```")
- yield from update_ui(chatbot=chatbot, history=history)
- raise RuntimeError('对话错误')
- if not gpt_replying_buffer:
- history = history[-2] # 错误的不纳入对话
- chatbot[-1] = (inputs, gpt_replying_buffer + f"触发了Google的安全访问策略，没有回答\n\n```\n{gpt_security_policy}\n```")
- yield from update_ui(chatbot=chatbot, history=history)
-
-
-
-if __name__ == '__main__':
- import sys
- llm_kwargs = {'llm_model': 'gemini-pro'}
- result = predict('Write long a story about a magic backpack.', llm_kwargs, llm_kwargs, [])
- for i in result:
- print(i)
diff --git a/request_llms/bridge_newbingfree.py b/request_llms/bridge_newbingfree.py
deleted file mode 100644
index 13573ede0977880c04547c37ca334a29d5e272fa..0000000000000000000000000000000000000000
--- a/request_llms/bridge_newbingfree.py
+++ /dev/null
@@ -1,311 +0,0 @@
-"""
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
-第一部分：来自EdgeGPT.py
-https://github.com/acheong08/EdgeGPT
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
-"""
-from .edge_gpt_free import Chatbot as NewbingChatbot
-
-load_message = "等待NewBing响应。"
-
-"""
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
-第二部分：子进程Worker（调用主体）
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
-"""
-import time
-import json
-import re
-import logging
-import asyncio
-import importlib
-import threading
-from toolbox import update_ui, get_conf, trimmed_format_exc
-from multiprocessing import Process, Pipe
-
-
-def preprocess_newbing_out(s):
- pattern = r"\^(\d+)\^" # 匹配^数字^
- sub = lambda m: "(" + m.group(1) + ")" # 将匹配到的数字作为替换值
- result = re.sub(pattern, sub, s) # 替换操作
- if "[1]" in result:
- result += (
- "\n\n```reference\n"
- + "\n".join([r for r in result.split("\n") if r.startswith("[")])
- + "\n```\n"
- )
- return result
-
-
-def preprocess_newbing_out_simple(result):
- if "[1]" in result:
- result += (
- "\n\n```reference\n"
- + "\n".join([r for r in result.split("\n") if r.startswith("[")])
- + "\n```\n"
- )
- return result
-
-
-class NewBingHandle(Process):
- def __init__(self):
- super().__init__(daemon=True)
- self.parent, self.child = Pipe()
- self.newbing_model = None
- self.info = ""
- self.success = True
- self.local_history = []
- self.check_dependency()
- self.start()
- self.threadLock = threading.Lock()
-
- def check_dependency(self):
- try:
- self.success = False
- import certifi, httpx, rich
-
- self.info = "依赖检测通过，等待NewBing响应。注意目前不能多人同时调用NewBing接口（有线程锁），否则将导致每个人的NewBing问询历史互相渗透。调用NewBing时，会自动使用已配置的代理。"
- self.success = True
- except:
- self.info = "缺少的依赖，如果要使用Newbing，除了基础的pip依赖以外，您还需要运行`pip install -r request_llms/requirements_newbing.txt`安装Newbing的依赖。"
- self.success = False
-
- def ready(self):
- return self.newbing_model is not None
-
- async def async_run(self):
- # 读取配置
- NEWBING_STYLE = get_conf("NEWBING_STYLE")
- from request_llms.bridge_all import model_info
-
- endpoint = model_info["newbing"]["endpoint"]
- while True:
- # 等待
- kwargs = self.child.recv()
- question = kwargs["query"]
- history = kwargs["history"]
- system_prompt = kwargs["system_prompt"]
-
- # 是否重置
- if len(self.local_history) > 0 and len(history) == 0:
- await self.newbing_model.reset()
- self.local_history = []
-
- # 开始问问题
- prompt = ""
- if system_prompt not in self.local_history:
- self.local_history.append(system_prompt)
- prompt += system_prompt + "\n"
-
- # 追加历史
- for ab in history:
- a, b = ab
- if a not in self.local_history:
- self.local_history.append(a)
- prompt += a + "\n"
-
- # 问题
- prompt += question
- self.local_history.append(question)
- print("question:", prompt)
- # 提交
- async for final, response in self.newbing_model.ask_stream(
- prompt=question,
- conversation_style=NEWBING_STYLE, # ["creative", "balanced", "precise"]
- wss_link=endpoint, # "wss://sydney.bing.com/sydney/ChatHub"
- ):
- if not final:
- print(response)
- self.child.send(str(response))
- else:
- print("-------- receive final ---------")
- self.child.send("[Finish]")
- # self.local_history.append(response)
-
- def run(self):
- """
- 这个函数运行在子进程
- """
- # 第一次运行，加载参数
- self.success = False
- self.local_history = []
- if (self.newbing_model is None) or (not self.success):
- # 代理设置
- proxies, NEWBING_COOKIES = get_conf("proxies", "NEWBING_COOKIES")
- if proxies is None:
- self.proxies_https = None
- else:
- self.proxies_https = proxies["https"]
-
- if (NEWBING_COOKIES is not None) and len(NEWBING_COOKIES) > 100:
- try:
- cookies = json.loads(NEWBING_COOKIES)
- except:
- self.success = False
- tb_str = "\n```\n" + trimmed_format_exc() + "\n```\n"
- self.child.send(f"[Local Message] NEWBING_COOKIES未填写或有格式错误。")
- self.child.send("[Fail]")
- self.child.send("[Finish]")
- raise RuntimeError(f"NEWBING_COOKIES未填写或有格式错误。")
- else:
- cookies = None
-
- try:
- self.newbing_model = NewbingChatbot(
- proxy=self.proxies_https, cookies=cookies
- )
- except:
- self.success = False
- tb_str = "\n```\n" + trimmed_format_exc() + "\n```\n"
- self.child.send(
- f"[Local Message] 不能加载Newbing组件，请注意Newbing组件已不再维护。{tb_str}"
- )
- self.child.send("[Fail]")
- self.child.send("[Finish]")
- raise RuntimeError(f"不能加载Newbing组件，请注意Newbing组件已不再维护。")
-
- self.success = True
- try:
- # 进入任务等待状态
- asyncio.run(self.async_run())
- except Exception:
- tb_str = "\n```\n" + trimmed_format_exc() + "\n```\n"
- self.child.send(
- f"[Local Message] Newbing 请求失败，报错信息如下. 如果是与网络相关的问题，建议更换代理协议（推荐http）或代理节点 {tb_str}."
- )
- self.child.send("[Fail]")
- self.child.send("[Finish]")
-
- def stream_chat(self, **kwargs):
- """
- 这个函数运行在主进程
- """
- self.threadLock.acquire() # 获取线程锁
- self.parent.send(kwargs) # 请求子进程
- while True:
- res = self.parent.recv() # 等待newbing回复的片段
- if res == "[Finish]":
- break # 结束
- elif res == "[Fail]":
- self.success = False
- break # 失败
- else:
- yield res # newbing回复的片段
- self.threadLock.release() # 释放线程锁
-
-
-"""
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
-第三部分：主进程统一调用函数接口
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
-"""
-global newbingfree_handle
-newbingfree_handle = None
-
-
-def predict_no_ui_long_connection(
- inputs,
- llm_kwargs,
- history=[],
- sys_prompt="",
- observe_window=[],
- console_slience=False,
-):
- """
- 多线程方法
- 函数的说明请见 request_llms/bridge_all.py
- """
- global newbingfree_handle
- if (newbingfree_handle is None) or (not newbingfree_handle.success):
- newbingfree_handle = NewBingHandle()
- if len(observe_window) >= 1:
- observe_window[0] = load_message + "\n\n" + newbingfree_handle.info
- if not newbingfree_handle.success:
- error = newbingfree_handle.info
- newbingfree_handle = None
- raise RuntimeError(error)
-
- # 没有 sys_prompt 接口，因此把prompt加入 history
- history_feedin = []
- for i in range(len(history) // 2):
- history_feedin.append([history[2 * i], history[2 * i + 1]])
-
- watch_dog_patience = 5 # 看门狗 (watchdog) 的耐心, 设置5秒即可
- response = ""
- if len(observe_window) >= 1:
- observe_window[0] = "[Local Message] 等待NewBing响应中 ..."
- for response in newbingfree_handle.stream_chat(
- query=inputs,
- history=history_feedin,
- system_prompt=sys_prompt,
- max_length=llm_kwargs["max_length"],
- top_p=llm_kwargs["top_p"],
- temperature=llm_kwargs["temperature"],
- ):
- if len(observe_window) >= 1:
- observe_window[0] = preprocess_newbing_out_simple(response)
- if len(observe_window) >= 2:
- if (time.time() - observe_window[1]) > watch_dog_patience:
- raise RuntimeError("程序终止。")
- return preprocess_newbing_out_simple(response)
-
-
-def predict(
- inputs,
- llm_kwargs,
- plugin_kwargs,
- chatbot,
- history=[],
- system_prompt="",
- stream=True,
- additional_fn=None,
-):
- """
- 单线程方法
- 函数的说明请见 request_llms/bridge_all.py
- """
- chatbot.append((inputs, "[Local Message] 等待NewBing响应中 ..."))
-
- global newbingfree_handle
- if (newbingfree_handle is None) or (not newbingfree_handle.success):
- newbingfree_handle = NewBingHandle()
- chatbot[-1] = (inputs, load_message + "\n\n" + newbingfree_handle.info)
- yield from update_ui(chatbot=chatbot, history=[])
- if not newbingfree_handle.success:
- newbingfree_handle = None
- return
-
- if additional_fn is not None:
- from core_functional import handle_core_functionality
-
- inputs, history = handle_core_functionality(
- additional_fn, inputs, history, chatbot
- )
-
- history_feedin = []
- for i in range(len(history) // 2):
- history_feedin.append([history[2 * i], history[2 * i + 1]])
-
- chatbot[-1] = (inputs, "[Local Message] 等待NewBing响应中 ...")
- response = "[Local Message] 等待NewBing响应中 ..."
- yield from update_ui(
- chatbot=chatbot, history=history, msg="NewBing响应缓慢，尚未完成全部响应，请耐心完成后再提交新问题。"
- )
- for response in newbingfree_handle.stream_chat(
- query=inputs,
- history=history_feedin,
- system_prompt=system_prompt,
- max_length=llm_kwargs["max_length"],
- top_p=llm_kwargs["top_p"],
- temperature=llm_kwargs["temperature"],
- ):
- chatbot[-1] = (inputs, preprocess_newbing_out(response))
- yield from update_ui(
- chatbot=chatbot, history=history, msg="NewBing响应缓慢，尚未完成全部响应，请耐心完成后再提交新问题。"
- )
- if response == "[Local Message] 等待NewBing响应中 ...":
- response = "[Local Message] NewBing响应异常，请刷新界面重试 ..."
- history.extend([inputs, response])
- logging.info(f"[raw_input] {inputs}")
- logging.info(f"[response] {response}")
- yield from update_ui(chatbot=chatbot, history=history, msg="完成全部响应，请提交新问题。")
diff --git a/request_llms/bridge_qwen.py b/request_llms/bridge_qwen.py
deleted file mode 100644
index 808c2c75c42b03c108374324ea9b775ab19c378b..0000000000000000000000000000000000000000
--- a/request_llms/bridge_qwen.py
+++ /dev/null
@@ -1,63 +0,0 @@
-import time
-import os
-from toolbox import update_ui, get_conf, update_ui_lastest_msg
-from toolbox import check_packages, report_exception
-
-model_name = 'Qwen'
-
-def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="", observe_window=[], console_slience=False):
- """
- ⭐多线程方法
- 函数的说明请见 request_llms/bridge_all.py
- """
- watch_dog_patience = 5
- response = ""
-
- from .com_qwenapi import QwenRequestInstance
- sri = QwenRequestInstance()
- for response in sri.generate(inputs, llm_kwargs, history, sys_prompt):
- if len(observe_window) >= 1:
- observe_window[0] = response
- if len(observe_window) >= 2:
- if (time.time()-observe_window[1]) > watch_dog_patience: raise RuntimeError("程序终止。")
- return response
-
-def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_prompt='', stream = True, additional_fn=None):
- """
- ⭐单线程方法
- 函数的说明请见 request_llms/bridge_all.py
- """
- chatbot.append((inputs, ""))
- yield from update_ui(chatbot=chatbot, history=history)
-
- # 尝试导入依赖，如果缺少依赖，则给出安装建议
- try:
- check_packages(["dashscope"])
- except:
- yield from update_ui_lastest_msg(f"导入软件依赖失败。使用该模型需要额外依赖，安装方法```pip install --upgrade dashscope```。",
- chatbot=chatbot, history=history, delay=0)
- return
-
- # 检查DASHSCOPE_API_KEY
- if get_conf("DASHSCOPE_API_KEY") == "":
- yield from update_ui_lastest_msg(f"请配置 DASHSCOPE_API_KEY。",
- chatbot=chatbot, history=history, delay=0)
- return
-
- if additional_fn is not None:
- from core_functional import handle_core_functionality
- inputs, history = handle_core_functionality(additional_fn, inputs, history, chatbot)
-
- # 开始接收回复
- from .com_qwenapi import QwenRequestInstance
- sri = QwenRequestInstance()
- response = f"[Local Message] 等待{model_name}响应中 ..."
- for response in sri.generate(inputs, llm_kwargs, history, system_prompt):
- chatbot[-1] = (inputs, response)
- yield from update_ui(chatbot=chatbot, history=history)
-
- # 总结输出
- if response == f"[Local Message] 等待{model_name}响应中 ...":
- response = f"[Local Message] {model_name}响应异常 ..."
- history.extend([inputs, response])
- yield from update_ui(chatbot=chatbot, history=history)
\ No newline at end of file
diff --git a/request_llms/bridge_skylark2.py b/request_llms/bridge_skylark2.py
deleted file mode 100644
index 1a8edcbca842ce642a3abe68813593754e307487..0000000000000000000000000000000000000000
--- a/request_llms/bridge_skylark2.py
+++ /dev/null
@@ -1,68 +0,0 @@
-import time
-from toolbox import update_ui, get_conf, update_ui_lastest_msg
-from toolbox import check_packages, report_exception
-
-model_name = '云雀大模型'
-
-def validate_key():
- YUNQUE_SECRET_KEY = get_conf("YUNQUE_SECRET_KEY")
- if YUNQUE_SECRET_KEY == '': return False
- return True
-
-def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="", observe_window=[], console_slience=False):
- """
- ⭐ 多线程方法
- 函数的说明请见 request_llms/bridge_all.py
- """
- watch_dog_patience = 5
- response = ""
-
- if validate_key() is False:
- raise RuntimeError('请配置YUNQUE_SECRET_KEY')
-
- from .com_skylark2api import YUNQUERequestInstance
- sri = YUNQUERequestInstance()
- for response in sri.generate(inputs, llm_kwargs, history, sys_prompt):
- if len(observe_window) >= 1:
- observe_window[0] = response
- if len(observe_window) >= 2:
- if (time.time()-observe_window[1]) > watch_dog_patience: raise RuntimeError("程序终止。")
- return response
-
-def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_prompt='', stream = True, additional_fn=None):
- """
- ⭐ 单线程方法
- 函数的说明请见 request_llms/bridge_all.py
- """
- chatbot.append((inputs, ""))
- yield from update_ui(chatbot=chatbot, history=history)
-
- # 尝试导入依赖，如果缺少依赖，则给出安装建议
- try:
- check_packages(["zhipuai"])
- except:
- yield from update_ui_lastest_msg(f"导入软件依赖失败。使用该模型需要额外依赖，安装方法```pip install --upgrade zhipuai```。",
- chatbot=chatbot, history=history, delay=0)
- return
-
- if validate_key() is False:
- yield from update_ui_lastest_msg(lastmsg="[Local Message] 请配置HUOSHAN_API_KEY", chatbot=chatbot, history=history, delay=0)
- return
-
- if additional_fn is not None:
- from core_functional import handle_core_functionality
- inputs, history = handle_core_functionality(additional_fn, inputs, history, chatbot)
-
- # 开始接收回复
- from .com_skylark2api import YUNQUERequestInstance
- sri = YUNQUERequestInstance()
- response = f"[Local Message] 等待{model_name}响应中 ..."
- for response in sri.generate(inputs, llm_kwargs, history, system_prompt):
- chatbot[-1] = (inputs, response)
- yield from update_ui(chatbot=chatbot, history=history)
-
- # 总结输出
- if response == f"[Local Message] 等待{model_name}响应中 ...":
- response = f"[Local Message] {model_name}响应异常 ..."
- history.extend([inputs, response])
- yield from update_ui(chatbot=chatbot, history=history)
\ No newline at end of file
diff --git a/request_llms/bridge_zhipu.py b/request_llms/bridge_zhipu.py
deleted file mode 100644
index ecb3b7550e499c73a784acd3f966fb7f635bb1ed..0000000000000000000000000000000000000000
--- a/request_llms/bridge_zhipu.py
+++ /dev/null
@@ -1,93 +0,0 @@
-import time
-import os
-from toolbox import update_ui, get_conf, update_ui_lastest_msg
-from toolbox import check_packages, report_exception, have_any_recent_upload_image_files
-
-model_name = '智谱AI大模型'
-zhipuai_default_model = 'glm-4'
-
-def validate_key():
- ZHIPUAI_API_KEY = get_conf("ZHIPUAI_API_KEY")
- if ZHIPUAI_API_KEY == '': return False
- return True
-
-def make_media_input(inputs, image_paths):
- for image_path in image_paths:
- inputs = inputs + f'
'
- return inputs
-
-def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="", observe_window=[], console_slience=False):
- """
- ⭐多线程方法
- 函数的说明请见 request_llms/bridge_all.py
- """
- watch_dog_patience = 5
- response = ""
-
- if llm_kwargs["llm_model"] == "zhipuai":
- llm_kwargs["llm_model"] = zhipuai_default_model
-
- if validate_key() is False:
- raise RuntimeError('请配置ZHIPUAI_API_KEY')
-
- # 开始接收回复
- from .com_zhipuglm import ZhipuChatInit
- zhipu_bro_init = ZhipuChatInit()
- for chunk, response in zhipu_bro_init.generate_chat(inputs, llm_kwargs, history, sys_prompt):
- if len(observe_window) >= 1:
- observe_window[0] = response
- if len(observe_window) >= 2:
- if (time.time() - observe_window[1]) > watch_dog_patience:
- raise RuntimeError("程序终止。")
- return response
-
-
-def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_prompt='', stream=True, additional_fn=None):
- """
- ⭐单线程方法
- 函数的说明请见 request_llms/bridge_all.py
- """
- chatbot.append([inputs, ""])
- yield from update_ui(chatbot=chatbot, history=history)
-
- # 尝试导入依赖，如果缺少依赖，则给出安装建议
- try:
- check_packages(["zhipuai"])
- except:
- yield from update_ui_lastest_msg(f"导入软件依赖失败。使用该模型需要额外依赖，安装方法```pip install --upgrade zhipuai```。",
- chatbot=chatbot, history=history, delay=0)
- return
-
- if validate_key() is False:
- yield from update_ui_lastest_msg(lastmsg="[Local Message] 请配置ZHIPUAI_API_KEY", chatbot=chatbot, history=history, delay=0)
- return
-
- if additional_fn is not None:
- from core_functional import handle_core_functionality
- inputs, history = handle_core_functionality(additional_fn, inputs, history, chatbot)
- chatbot[-1] = [inputs, ""]
- yield from update_ui(chatbot=chatbot, history=history)
-
- if llm_kwargs["llm_model"] == "zhipuai":
- llm_kwargs["llm_model"] = zhipuai_default_model
-
- if llm_kwargs["llm_model"] in ["glm-4v"]:
- have_recent_file, image_paths = have_any_recent_upload_image_files(chatbot)
- if not have_recent_file:
- chatbot.append((inputs, "没有检测到任何近期上传的图像文件，请上传jpg格式的图片，此外，请注意拓展名需要小写"))
- yield from update_ui(chatbot=chatbot, history=history, msg="等待图片") # 刷新界面
- return
- if have_recent_file:
- inputs = make_media_input(inputs, image_paths)
- chatbot[-1] = [inputs, ""]
- yield from update_ui(chatbot=chatbot, history=history)
-
-
- # 开始接收回复
- from .com_zhipuglm import ZhipuChatInit
- zhipu_bro_init = ZhipuChatInit()
- for chunk, response in zhipu_bro_init.generate_chat(inputs, llm_kwargs, history, system_prompt):
- chatbot[-1] = [inputs, response]
- yield from update_ui(chatbot=chatbot, history=history)
- history.extend([inputs, response])
- yield from update_ui(chatbot=chatbot, history=history)
\ No newline at end of file
diff --git a/request_llms/com_google.py b/request_llms/com_google.py
deleted file mode 100644
index e66d659af5a0fb99ff803162f4fd0b6e5505ee29..0000000000000000000000000000000000000000
--- a/request_llms/com_google.py
+++ /dev/null
@@ -1,201 +0,0 @@
-# encoding: utf-8
-# @Time : 2023/12/25
-# @Author : Spike
-# @Descr :
-import json
-import os
-import re
-import requests
-from typing import List, Dict, Tuple
-from toolbox import get_conf, encode_image, get_pictures_list, to_markdown_tabs
-
-proxies, TIMEOUT_SECONDS = get_conf("proxies", "TIMEOUT_SECONDS")
-
-"""
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
-第五部分 一些文件处理方法
-files_filter_handler 根据type过滤文件
-input_encode_handler 提取input中的文件，并解析
-file_manifest_filter_html 根据type过滤文件, 并解析为html or md 文本
-link_mtime_to_md 文件增加本地时间参数，避免下载到缓存文件
-html_view_blank 超链接
-html_local_file 本地文件取相对路径
-to_markdown_tabs 文件list 转换为 md tab
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
-"""
-
-
-def files_filter_handler(file_list):
- new_list = []
- filter_ = [
- "png",
- "jpg",
- "jpeg",
- "bmp",
- "svg",
- "webp",
- "ico",
- "tif",
- "tiff",
- "raw",
- "eps",
- ]
- for file in file_list:
- file = str(file).replace("file=", "")
- if os.path.exists(file):
- if str(os.path.basename(file)).split(".")[-1] in filter_:
- new_list.append(file)
- return new_list
-
-
-def input_encode_handler(inputs, llm_kwargs):
- if llm_kwargs["most_recent_uploaded"].get("path"):
- image_paths = get_pictures_list(llm_kwargs["most_recent_uploaded"]["path"])
- md_encode = []
- for md_path in image_paths:
- type_ = os.path.splitext(md_path)[1].replace(".", "")
- type_ = "jpeg" if type_ == "jpg" else type_
- md_encode.append({"data": encode_image(md_path), "type": type_})
- return inputs, md_encode
-
-
-def file_manifest_filter_html(file_list, filter_: list = None, md_type=False):
- new_list = []
- if not filter_:
- filter_ = [
- "png",
- "jpg",
- "jpeg",
- "bmp",
- "svg",
- "webp",
- "ico",
- "tif",
- "tiff",
- "raw",
- "eps",
- ]
- for file in file_list:
- if str(os.path.basename(file)).split(".")[-1] in filter_:
- new_list.append(html_local_img(file, md=md_type))
- elif os.path.exists(file):
- new_list.append(link_mtime_to_md(file))
- else:
- new_list.append(file)
- return new_list
-
-
-def link_mtime_to_md(file):
- link_local = html_local_file(file)
- link_name = os.path.basename(file)
- a = f"[{link_name}]({link_local}?{os.path.getmtime(file)})"
- return a
-
-
-def html_local_file(file):
- base_path = os.path.dirname(__file__) # 项目目录
- if os.path.exists(str(file)):
- file = f'file={file.replace(base_path, ".")}'
- return file
-
-
-def html_local_img(__file, layout="left", max_width=None, max_height=None, md=True):
- style = ""
- if max_width is not None:
- style += f"max-width: {max_width};"
- if max_height is not None:
- style += f"max-height: {max_height};"
- __file = html_local_file(__file)
- a = f''
- if md:
- a = f""
- return a
-
-
-
-class GoogleChatInit:
- def __init__(self):
- self.url_gemini = "https://generativelanguage.googleapis.com/v1beta/models/%m:streamGenerateContent?key=%k"
-
- def generate_chat(self, inputs, llm_kwargs, history, system_prompt):
- headers, payload = self.generate_message_payload(
- inputs, llm_kwargs, history, system_prompt
- )
- response = requests.post(
- url=self.url_gemini,
- headers=headers,
- data=json.dumps(payload),
- stream=True,
- proxies=proxies,
- timeout=TIMEOUT_SECONDS,
- )
- return response.iter_lines()
-
- def __conversation_user(self, user_input, llm_kwargs):
- what_i_have_asked = {"role": "user", "parts": []}
- if "vision" not in self.url_gemini:
- input_ = user_input
- encode_img = []
- else:
- input_, encode_img = input_encode_handler(user_input, llm_kwargs=llm_kwargs)
- what_i_have_asked["parts"].append({"text": input_})
- if encode_img:
- for data in encode_img:
- what_i_have_asked["parts"].append(
- {
- "inline_data": {
- "mime_type": f"image/{data['type']}",
- "data": data["data"],
- }
- }
- )
- return what_i_have_asked
-
- def __conversation_history(self, history, llm_kwargs):
- messages = []
- conversation_cnt = len(history) // 2
- if conversation_cnt:
- for index in range(0, 2 * conversation_cnt, 2):
- what_i_have_asked = self.__conversation_user(history[index], llm_kwargs)
- what_gpt_answer = {
- "role": "model",
- "parts": [{"text": history[index + 1]}],
- }
- messages.append(what_i_have_asked)
- messages.append(what_gpt_answer)
- return messages
-
- def generate_message_payload(
- self, inputs, llm_kwargs, history, system_prompt
- ) -> Tuple[Dict, Dict]:
- messages = [
- # {"role": "system", "parts": [{"text": system_prompt}]}, # gemini 不允许对话轮次为偶数，所以这个没有用，看后续支持吧。。。
- # {"role": "user", "parts": [{"text": ""}]},
- # {"role": "model", "parts": [{"text": ""}]}
- ]
- self.url_gemini = self.url_gemini.replace(
- "%m", llm_kwargs["llm_model"]
- ).replace("%k", get_conf("GEMINI_API_KEY"))
- header = {"Content-Type": "application/json"}
- if "vision" not in self.url_gemini: # 不是vision 才处理history
- messages.extend(
- self.__conversation_history(history, llm_kwargs)
- ) # 处理 history
- messages.append(self.__conversation_user(inputs, llm_kwargs)) # 处理用户对话
- payload = {
- "contents": messages,
- "generationConfig": {
- # "maxOutputTokens": 800,
- "stopSequences": str(llm_kwargs.get("stop", "")).split(" "),
- "temperature": llm_kwargs.get("temperature", 1),
- "topP": llm_kwargs.get("top_p", 0.8),
- "topK": 10,
- },
- }
- return header, payload
-
-
-if __name__ == "__main__":
- google = GoogleChatInit()
- # print(gootle.generate_message_payload('你好呀', {}, ['123123', '3123123'], ''))
- # gootle.input_encode_handle('123123[123123](./123123), ')
diff --git a/request_llms/com_qwenapi.py b/request_llms/com_qwenapi.py
deleted file mode 100644
index 5807600d3e94607f3eee96b758f3fda04a848dd0..0000000000000000000000000000000000000000
--- a/request_llms/com_qwenapi.py
+++ /dev/null
@@ -1,94 +0,0 @@
-from http import HTTPStatus
-from toolbox import get_conf
-import threading
-import logging
-
-timeout_bot_msg = '[Local Message] Request timeout. Network error.'
-
-class QwenRequestInstance():
- def __init__(self):
- import dashscope
- self.time_to_yield_event = threading.Event()
- self.time_to_exit_event = threading.Event()
- self.result_buf = ""
-
- def validate_key():
- DASHSCOPE_API_KEY = get_conf("DASHSCOPE_API_KEY")
- if DASHSCOPE_API_KEY == '': return False
- return True
-
- if not validate_key():
- raise RuntimeError('请配置 DASHSCOPE_API_KEY')
- dashscope.api_key = get_conf("DASHSCOPE_API_KEY")
-
-
- def generate(self, inputs, llm_kwargs, history, system_prompt):
- # import _thread as thread
- from dashscope import Generation
- QWEN_MODEL = {
- 'qwen-turbo': Generation.Models.qwen_turbo,
- 'qwen-plus': Generation.Models.qwen_plus,
- 'qwen-max': Generation.Models.qwen_max,
- }[llm_kwargs['llm_model']]
- top_p = llm_kwargs.get('top_p', 0.8)
- if top_p == 0: top_p += 1e-5
- if top_p == 1: top_p -= 1e-5
-
- self.result_buf = ""
- responses = Generation.call(
- model=QWEN_MODEL,
- messages=generate_message_payload(inputs, llm_kwargs, history, system_prompt),
- top_p=top_p,
- temperature=llm_kwargs.get('temperature', 1.0),
- result_format='message',
- stream=True,
- incremental_output=True
- )
-
- for response in responses:
- if response.status_code == HTTPStatus.OK:
- if response.output.choices[0].finish_reason == 'stop':
- yield self.result_buf
- break
- elif response.output.choices[0].finish_reason == 'length':
- self.result_buf += "[Local Message] 生成长度过长，后续输出被截断"
- yield self.result_buf
- break
- else:
- self.result_buf += response.output.choices[0].message.content
- yield self.result_buf
- else:
- self.result_buf += f"[Local Message] 请求错误：状态码：{response.status_code}，错误码:{response.code}，消息：{response.message}"
- yield self.result_buf
- break
- logging.info(f'[raw_input] {inputs}')
- logging.info(f'[response] {self.result_buf}')
- return self.result_buf
-
-
-def generate_message_payload(inputs, llm_kwargs, history, system_prompt):
- conversation_cnt = len(history) // 2
- if system_prompt == '': system_prompt = 'Hello!'
- messages = [{"role": "user", "content": system_prompt}, {"role": "assistant", "content": "Certainly!"}]
- if conversation_cnt:
- for index in range(0, 2*conversation_cnt, 2):
- what_i_have_asked = {}
- what_i_have_asked["role"] = "user"
- what_i_have_asked["content"] = history[index]
- what_gpt_answer = {}
- what_gpt_answer["role"] = "assistant"
- what_gpt_answer["content"] = history[index+1]
- if what_i_have_asked["content"] != "":
- if what_gpt_answer["content"] == "":
- continue
- if what_gpt_answer["content"] == timeout_bot_msg:
- continue
- messages.append(what_i_have_asked)
- messages.append(what_gpt_answer)
- else:
- messages[-1]['content'] = what_gpt_answer['content']
- what_i_ask_now = {}
- what_i_ask_now["role"] = "user"
- what_i_ask_now["content"] = inputs
- messages.append(what_i_ask_now)
- return messages
diff --git a/request_llms/com_skylark2api.py b/request_llms/com_skylark2api.py
deleted file mode 100644
index 2530eccaab9e05f10f16fabe3293103b52058f0e..0000000000000000000000000000000000000000
--- a/request_llms/com_skylark2api.py
+++ /dev/null
@@ -1,95 +0,0 @@
-from toolbox import get_conf
-import threading
-import logging
-import os
-
-timeout_bot_msg = '[Local Message] Request timeout. Network error.'
-#os.environ['VOLC_ACCESSKEY'] = ''
-#os.environ['VOLC_SECRETKEY'] = ''
-
-class YUNQUERequestInstance():
- def __init__(self):
-
- self.time_to_yield_event = threading.Event()
- self.time_to_exit_event = threading.Event()
-
- self.result_buf = ""
-
- def generate(self, inputs, llm_kwargs, history, system_prompt):
- # import _thread as thread
- from volcengine.maas import MaasService, MaasException
-
- maas = MaasService('maas-api.ml-platform-cn-beijing.volces.com', 'cn-beijing')
-
- YUNQUE_SECRET_KEY, YUNQUE_ACCESS_KEY,YUNQUE_MODEL = get_conf("YUNQUE_SECRET_KEY", "YUNQUE_ACCESS_KEY","YUNQUE_MODEL")
- maas.set_ak(YUNQUE_ACCESS_KEY) #填写 VOLC_ACCESSKEY
- maas.set_sk(YUNQUE_SECRET_KEY) #填写 'VOLC_SECRETKEY'
-
- self.result_buf = ""
-
- req = {
- "model": {
- "name": YUNQUE_MODEL,
- "version": "1.0", # use default version if not specified.
- },
- "parameters": {
- "max_new_tokens": 4000, # 输出文本的最大tokens限制
- "min_new_tokens": 1, # 输出文本的最小tokens限制
- "temperature": llm_kwargs['temperature'], # 用于控制生成文本的随机性和创造性，Temperature值越大随机性越大，取值范围0~1
- "top_p": llm_kwargs['top_p'], # 用于控制输出tokens的多样性，TopP值越大输出的tokens类型越丰富，取值范围0~1
- "top_k": 0, # 选择预测值最大的k个token进行采样，取值范围0-1000，0表示不生效
- "max_prompt_tokens": 4000, # 最大输入 token 数，如果给出的 prompt 的 token 长度超过此限制，取最后 max_prompt_tokens 个 token 输入模型。
- },
- "messages": self.generate_message_payload(inputs, llm_kwargs, history, system_prompt)
- }
-
- response = maas.stream_chat(req)
-
- for resp in response:
- self.result_buf += resp.choice.message.content
- yield self.result_buf
- '''
- for event in response.events():
- if event.event == "add":
- self.result_buf += event.data
- yield self.result_buf
- elif event.event == "error" or event.event == "interrupted":
- raise RuntimeError("Unknown error:" + event.data)
- elif event.event == "finish":
- yield self.result_buf
- break
- else:
- raise RuntimeError("Unknown error:" + str(event))
-
- logging.info(f'[raw_input] {inputs}')
- logging.info(f'[response] {self.result_buf}')
- '''
- return self.result_buf
-
- def generate_message_payload(inputs, llm_kwargs, history, system_prompt):
- from volcengine.maas import ChatRole
- conversation_cnt = len(history) // 2
- messages = [{"role": ChatRole.USER, "content": system_prompt},
- {"role": ChatRole.ASSISTANT, "content": "Certainly!"}]
- if conversation_cnt:
- for index in range(0, 2 * conversation_cnt, 2):
- what_i_have_asked = {}
- what_i_have_asked["role"] = ChatRole.USER
- what_i_have_asked["content"] = history[index]
- what_gpt_answer = {}
- what_gpt_answer["role"] = ChatRole.ASSISTANT
- what_gpt_answer["content"] = history[index + 1]
- if what_i_have_asked["content"] != "":
- if what_gpt_answer["content"] == "":
- continue
- if what_gpt_answer["content"] == timeout_bot_msg:
- continue
- messages.append(what_i_have_asked)
- messages.append(what_gpt_answer)
- else:
- messages[-1]['content'] = what_gpt_answer['content']
- what_i_ask_now = {}
- what_i_ask_now["role"] = ChatRole.USER
- what_i_ask_now["content"] = inputs
- messages.append(what_i_ask_now)
- return messages
\ No newline at end of file
diff --git a/request_llms/com_zhipuapi.py b/request_llms/com_zhipuapi.py
deleted file mode 100644
index d8b763c9087d4b505c2a75f277ec4e1c49de3edd..0000000000000000000000000000000000000000
--- a/request_llms/com_zhipuapi.py
+++ /dev/null
@@ -1,70 +0,0 @@
-from toolbox import get_conf
-import threading
-import logging
-
-timeout_bot_msg = '[Local Message] Request timeout. Network error.'
-
-class ZhipuRequestInstance():
- def __init__(self):
-
- self.time_to_yield_event = threading.Event()
- self.time_to_exit_event = threading.Event()
-
- self.result_buf = ""
-
- def generate(self, inputs, llm_kwargs, history, system_prompt):
- # import _thread as thread
- import zhipuai
- ZHIPUAI_API_KEY, ZHIPUAI_MODEL = get_conf("ZHIPUAI_API_KEY", "ZHIPUAI_MODEL")
- zhipuai.api_key = ZHIPUAI_API_KEY
- self.result_buf = ""
- response = zhipuai.model_api.sse_invoke(
- model=ZHIPUAI_MODEL,
- prompt=generate_message_payload(inputs, llm_kwargs, history, system_prompt),
- top_p=llm_kwargs['top_p']*0.7, # 智谱的API抽风，手动*0.7给做个线性变换
- temperature=llm_kwargs['temperature']*0.95, # 智谱的API抽风，手动*0.7给做个线性变换
- )
- for event in response.events():
- if event.event == "add":
- # if self.result_buf == "" and event.data.startswith(" "):
- # event.data = event.data.lstrip(" ") # 每次智谱为啥都要带个空格开头呢？
- self.result_buf += event.data
- yield self.result_buf
- elif event.event == "error" or event.event == "interrupted":
- raise RuntimeError("Unknown error:" + event.data)
- elif event.event == "finish":
- yield self.result_buf
- break
- else:
- raise RuntimeError("Unknown error:" + str(event))
- if self.result_buf == "":
- yield "智谱没有返回任何数据, 请检查ZHIPUAI_API_KEY和ZHIPUAI_MODEL是否填写正确."
- logging.info(f'[raw_input] {inputs}')
- logging.info(f'[response] {self.result_buf}')
- return self.result_buf
-
-def generate_message_payload(inputs, llm_kwargs, history, system_prompt):
- conversation_cnt = len(history) // 2
- messages = [{"role": "user", "content": system_prompt}, {"role": "assistant", "content": "Certainly!"}]
- if conversation_cnt:
- for index in range(0, 2*conversation_cnt, 2):
- what_i_have_asked = {}
- what_i_have_asked["role"] = "user"
- what_i_have_asked["content"] = history[index]
- what_gpt_answer = {}
- what_gpt_answer["role"] = "assistant"
- what_gpt_answer["content"] = history[index+1]
- if what_i_have_asked["content"] != "":
- if what_gpt_answer["content"] == "":
- continue
- if what_gpt_answer["content"] == timeout_bot_msg:
- continue
- messages.append(what_i_have_asked)
- messages.append(what_gpt_answer)
- else:
- messages[-1]['content'] = what_gpt_answer['content']
- what_i_ask_now = {}
- what_i_ask_now["role"] = "user"
- what_i_ask_now["content"] = inputs
- messages.append(what_i_ask_now)
- return messages
diff --git a/request_llms/com_zhipuglm.py b/request_llms/com_zhipuglm.py
deleted file mode 100644
index 2e96d3fd87b1759b0dd52206b5dc03b7b760aa52..0000000000000000000000000000000000000000
--- a/request_llms/com_zhipuglm.py
+++ /dev/null
@@ -1,84 +0,0 @@
-# encoding: utf-8
-# @Time : 2024/1/22
-# @Author : Kilig947 & binary husky
-# @Descr : 兼容最新的智谱Ai
-from toolbox import get_conf
-from zhipuai import ZhipuAI
-from toolbox import get_conf, encode_image, get_pictures_list
-import logging, os
-
-
-def input_encode_handler(inputs, llm_kwargs):
- if llm_kwargs["most_recent_uploaded"].get("path"):
- image_paths = get_pictures_list(llm_kwargs["most_recent_uploaded"]["path"])
- md_encode = []
- for md_path in image_paths:
- type_ = os.path.splitext(md_path)[1].replace(".", "")
- type_ = "jpeg" if type_ == "jpg" else type_
- md_encode.append({"data": encode_image(md_path), "type": type_})
- return inputs, md_encode
-
-
-class ZhipuChatInit:
-
- def __init__(self):
- ZHIPUAI_API_KEY, ZHIPUAI_MODEL = get_conf("ZHIPUAI_API_KEY", "ZHIPUAI_MODEL")
- if len(ZHIPUAI_MODEL) > 0:
- logging.error('ZHIPUAI_MODEL 配置项选项已经弃用，请在LLM_MODEL中配置')
- self.zhipu_bro = ZhipuAI(api_key=ZHIPUAI_API_KEY)
- self.model = ''
-
- def __conversation_user(self, user_input: str, llm_kwargs):
- if self.model not in ["glm-4v"]:
- return {"role": "user", "content": user_input}
- else:
- input_, encode_img = input_encode_handler(user_input, llm_kwargs=llm_kwargs)
- what_i_have_asked = {"role": "user", "content": []}
- what_i_have_asked['content'].append({"type": 'text', "text": user_input})
- if encode_img:
- img_d = {"type": "image_url",
- "image_url": {'url': encode_img}}
- what_i_have_asked['content'].append(img_d)
- return what_i_have_asked
-
- def __conversation_history(self, history, llm_kwargs):
- messages = []
- conversation_cnt = len(history) // 2
- if conversation_cnt:
- for index in range(0, 2 * conversation_cnt, 2):
- what_i_have_asked = self.__conversation_user(history[index], llm_kwargs)
- what_gpt_answer = {
- "role": "assistant",
- "content": history[index + 1]
- }
- messages.append(what_i_have_asked)
- messages.append(what_gpt_answer)
- return messages
-
- def __conversation_message_payload(self, inputs, llm_kwargs, history, system_prompt):
- messages = []
- if system_prompt:
- messages.append({"role": "system", "content": system_prompt})
- self.model = llm_kwargs['llm_model']
- messages.extend(self.__conversation_history(history, llm_kwargs)) # 处理 history
- messages.append(self.__conversation_user(inputs, llm_kwargs)) # 处理用户对话
- response = self.zhipu_bro.chat.completions.create(
- model=self.model, messages=messages, stream=True,
- temperature=llm_kwargs.get('temperature', 0.95) * 0.95, # 只能传默认的 temperature 和 top_p
- top_p=llm_kwargs.get('top_p', 0.7) * 0.7,
- max_tokens=llm_kwargs.get('max_tokens', 1024 * 4), # 最大输出模型的一半
- )
- return response
-
- def generate_chat(self, inputs, llm_kwargs, history, system_prompt):
- self.model = llm_kwargs['llm_model']
- response = self.__conversation_message_payload(inputs, llm_kwargs, history, system_prompt)
- bro_results = ''
- for chunk in response:
- bro_results += chunk.choices[0].delta.content
- yield chunk.choices[0].delta.content, bro_results
-
-
-if __name__ == '__main__':
- zhipu = ZhipuChatInit()
- zhipu.generate_chat('你好', {'llm_model': 'glm-4'}, [], '你是WPSAi')
diff --git a/request_llms/key_manager.py b/request_llms/key_manager.py
deleted file mode 100644
index 8563d2ef823e79cb9242ca924ac5b525dc21fffb..0000000000000000000000000000000000000000
--- a/request_llms/key_manager.py
+++ /dev/null
@@ -1,29 +0,0 @@
-import random
-
-def Singleton(cls):
- _instance = {}
-
- def _singleton(*args, **kargs):
- if cls not in _instance:
- _instance[cls] = cls(*args, **kargs)
- return _instance[cls]
-
- return _singleton
-
-
-@Singleton
-class OpenAI_ApiKeyManager():
- def __init__(self, mode='blacklist') -> None:
- # self.key_avail_list = []
- self.key_black_list = []
-
- def add_key_to_blacklist(self, key):
- self.key_black_list.append(key)
-
- def select_avail_key(self, key_list):
- # select key from key_list, but avoid keys also in self.key_black_list, raise error if no key can be found
- available_keys = [key for key in key_list if key not in self.key_black_list]
- if not available_keys:
- raise KeyError("No available key found.")
- selected_key = random.choice(available_keys)
- return selected_key
\ No newline at end of file
diff --git a/request_llms/local_llm_class.py b/request_llms/local_llm_class.py
deleted file mode 100644
index ec7cfd2195211587691628094a803e8432714ee9..0000000000000000000000000000000000000000
--- a/request_llms/local_llm_class.py
+++ /dev/null
@@ -1,319 +0,0 @@
-import time
-import threading
-from toolbox import update_ui, Singleton
-from multiprocessing import Process, Pipe
-from contextlib import redirect_stdout
-from request_llms.queued_pipe import create_queue_pipe
-
-class ThreadLock(object):
- def __init__(self):
- self._lock = threading.Lock()
-
- def acquire(self):
- # print("acquiring", self)
- #traceback.print_tb
- self._lock.acquire()
- # print("acquired", self)
-
- def release(self):
- # print("released", self)
- #traceback.print_tb
- self._lock.release()
-
- def __enter__(self):
- self.acquire()
-
- def __exit__(self, type, value, traceback):
- self.release()
-
-@Singleton
-class GetSingletonHandle():
- def __init__(self):
- self.llm_model_already_running = {}
-
- def get_llm_model_instance(self, cls, *args, **kargs):
- if cls not in self.llm_model_already_running:
- self.llm_model_already_running[cls] = cls(*args, **kargs)
- return self.llm_model_already_running[cls]
- elif self.llm_model_already_running[cls].corrupted:
- self.llm_model_already_running[cls] = cls(*args, **kargs)
- return self.llm_model_already_running[cls]
- else:
- return self.llm_model_already_running[cls]
-
-def reset_tqdm_output():
- import sys, tqdm
- def status_printer(self, file):
- fp = file
- if fp in (sys.stderr, sys.stdout):
- getattr(sys.stderr, 'flush', lambda: None)()
- getattr(sys.stdout, 'flush', lambda: None)()
-
- def fp_write(s):
- print(s)
- last_len = [0]
-
- def print_status(s):
- from tqdm.utils import disp_len
- len_s = disp_len(s)
- fp_write('\r' + s + (' ' * max(last_len[0] - len_s, 0)))
- last_len[0] = len_s
- return print_status
- tqdm.tqdm.status_printer = status_printer
-
-
-class LocalLLMHandle(Process):
- def __init__(self):
- # ⭐run in main process
- super().__init__(daemon=True)
- self.is_main_process = True # init
- self.corrupted = False
- self.load_model_info()
- self.parent, self.child = create_queue_pipe()
- self.parent_state, self.child_state = create_queue_pipe()
- # allow redirect_stdout
- self.std_tag = "[Subprocess Message] "
- self.running = True
- self._model = None
- self._tokenizer = None
- self.state = ""
- self.check_dependency()
- self.is_main_process = False # state wrap for child process
- self.start()
- self.is_main_process = True # state wrap for child process
- self.threadLock = ThreadLock()
-
- def get_state(self):
- # ⭐run in main process
- while self.parent_state.poll():
- self.state = self.parent_state.recv()
- return self.state
-
- def set_state(self, new_state):
- # ⭐run in main process or 🏃♂️🏃♂️🏃♂️ run in child process
- if self.is_main_process:
- self.state = new_state
- else:
- self.child_state.send(new_state)
-
- def load_model_info(self):
- # 🏃♂️🏃♂️🏃♂️ run in child process
- raise NotImplementedError("Method not implemented yet")
- self.model_name = ""
- self.cmd_to_install = ""
-
- def load_model_and_tokenizer(self):
- """
- This function should return the model and the tokenizer
- """
- # 🏃♂️🏃♂️🏃♂️ run in child process
- raise NotImplementedError("Method not implemented yet")
-
- def llm_stream_generator(self, **kwargs):
- # 🏃♂️🏃♂️🏃♂️ run in child process
- raise NotImplementedError("Method not implemented yet")
-
- def try_to_import_special_deps(self, **kwargs):
- """
- import something that will raise error if the user does not install requirement_*.txt
- """
- # ⭐run in main process
- raise NotImplementedError("Method not implemented yet")
-
- def check_dependency(self):
- # ⭐run in main process
- try:
- self.try_to_import_special_deps()
- self.set_state("`依赖检测通过`")
- self.running = True
- except:
- self.set_state(f"缺少{self.model_name}的依赖，如果要使用{self.model_name}，除了基础的pip依赖以外，您还需要运行{self.cmd_to_install}安装{self.model_name}的依赖。")
- self.running = False
-
- def run(self):
- # 🏃♂️🏃♂️🏃♂️ run in child process
- # 第一次运行，加载参数
- self.child.flush = lambda *args: None
- self.child.write = lambda x: self.child.send(self.std_tag + x)
- reset_tqdm_output()
- self.set_state("`尝试加载模型`")
- try:
- with redirect_stdout(self.child):
- self._model, self._tokenizer = self.load_model_and_tokenizer()
- except:
- self.set_state("`加载模型失败`")
- self.running = False
- from toolbox import trimmed_format_exc
- self.child.send(
- f'[Local Message] 不能正常加载{self.model_name}的参数.' + '\n```\n' + trimmed_format_exc() + '\n```\n')
- self.child.send('[FinishBad]')
- raise RuntimeError(f"不能正常加载{self.model_name}的参数！")
-
- self.set_state("`准备就绪`")
- while True:
- # 进入任务等待状态
- kwargs = self.child.recv()
- # 收到消息，开始请求
- try:
- for response_full in self.llm_stream_generator(**kwargs):
- self.child.send(response_full)
- # print('debug' + response_full)
- self.child.send('[Finish]')
- # 请求处理结束，开始下一个循环
- except:
- from toolbox import trimmed_format_exc
- self.child.send(
- f'[Local Message] 调用{self.model_name}失败.' + '\n```\n' + trimmed_format_exc() + '\n```\n')
- self.child.send('[Finish]')
-
- def clear_pending_messages(self):
- # ⭐run in main process
- while True:
- if self.parent.poll():
- self.parent.recv()
- continue
- for _ in range(5):
- time.sleep(0.5)
- if self.parent.poll():
- r = self.parent.recv()
- continue
- break
- return
-
- def stream_chat(self, **kwargs):
- # ⭐run in main process
- if self.get_state() == "`准备就绪`":
- yield "`正在等待线程锁，排队中请稍候 ...`"
-
- with self.threadLock:
- if self.parent.poll():
- yield "`排队中请稍候 ...`"
- self.clear_pending_messages()
- self.parent.send(kwargs)
- std_out = ""
- std_out_clip_len = 4096
- while True:
- res = self.parent.recv()
- # pipe_watch_dog.feed()
- if res.startswith(self.std_tag):
- new_output = res[len(self.std_tag):]
- std_out = std_out[:std_out_clip_len]
- print(new_output, end='')
- std_out = new_output + std_out
- yield self.std_tag + '\n```\n' + std_out + '\n```\n'
- elif res == '[Finish]':
- break
- elif res == '[FinishBad]':
- self.running = False
- self.corrupted = True
- break
- else:
- std_out = ""
- yield res
-
-def get_local_llm_predict_fns(LLMSingletonClass, model_name, history_format='classic'):
- load_message = f"{model_name}尚未加载，加载需要一段时间。注意，取决于`config.py`的配置，{model_name}消耗大量的内存（CPU）或显存（GPU），也许会导致低配计算机卡死 ……"
-
- def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="", observe_window=[], console_slience=False):
- """
- refer to request_llms/bridge_all.py
- """
- _llm_handle = GetSingletonHandle().get_llm_model_instance(LLMSingletonClass)
- if len(observe_window) >= 1:
- observe_window[0] = load_message + "\n\n" + _llm_handle.get_state()
- if not _llm_handle.running:
- raise RuntimeError(_llm_handle.get_state())
-
- if history_format == 'classic':
- # 没有 sys_prompt 接口，因此把prompt加入 history
- history_feedin = []
- history_feedin.append([sys_prompt, "Certainly!"])
- for i in range(len(history)//2):
- history_feedin.append([history[2*i], history[2*i+1]])
- elif history_format == 'chatglm3':
- # 有 sys_prompt 接口
- conversation_cnt = len(history) // 2
- history_feedin = [{"role": "system", "content": sys_prompt}]
- if conversation_cnt:
- for index in range(0, 2*conversation_cnt, 2):
- what_i_have_asked = {}
- what_i_have_asked["role"] = "user"
- what_i_have_asked["content"] = history[index]
- what_gpt_answer = {}
- what_gpt_answer["role"] = "assistant"
- what_gpt_answer["content"] = history[index+1]
- if what_i_have_asked["content"] != "":
- if what_gpt_answer["content"] == "":
- continue
- history_feedin.append(what_i_have_asked)
- history_feedin.append(what_gpt_answer)
- else:
- history_feedin[-1]['content'] = what_gpt_answer['content']
-
- watch_dog_patience = 5 # 看门狗 (watchdog) 的耐心, 设置5秒即可
- response = ""
- for response in _llm_handle.stream_chat(query=inputs, history=history_feedin, max_length=llm_kwargs['max_length'], top_p=llm_kwargs['top_p'], temperature=llm_kwargs['temperature']):
- if len(observe_window) >= 1:
- observe_window[0] = response
- if len(observe_window) >= 2:
- if (time.time()-observe_window[1]) > watch_dog_patience:
- raise RuntimeError("程序终止。")
- return response
-
- def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_prompt='', stream=True, additional_fn=None):
- """
- refer to request_llms/bridge_all.py
- """
- chatbot.append((inputs, ""))
-
- _llm_handle = GetSingletonHandle().get_llm_model_instance(LLMSingletonClass)
- chatbot[-1] = (inputs, load_message + "\n\n" + _llm_handle.get_state())
- yield from update_ui(chatbot=chatbot, history=[])
- if not _llm_handle.running:
- raise RuntimeError(_llm_handle.get_state())
-
- if additional_fn is not None:
- from core_functional import handle_core_functionality
- inputs, history = handle_core_functionality(
- additional_fn, inputs, history, chatbot)
-
- # 处理历史信息
- if history_format == 'classic':
- # 没有 sys_prompt 接口，因此把prompt加入 history
- history_feedin = []
- history_feedin.append([system_prompt, "Certainly!"])
- for i in range(len(history)//2):
- history_feedin.append([history[2*i], history[2*i+1]])
- elif history_format == 'chatglm3':
- # 有 sys_prompt 接口
- conversation_cnt = len(history) // 2
- history_feedin = [{"role": "system", "content": system_prompt}]
- if conversation_cnt:
- for index in range(0, 2*conversation_cnt, 2):
- what_i_have_asked = {}
- what_i_have_asked["role"] = "user"
- what_i_have_asked["content"] = history[index]
- what_gpt_answer = {}
- what_gpt_answer["role"] = "assistant"
- what_gpt_answer["content"] = history[index+1]
- if what_i_have_asked["content"] != "":
- if what_gpt_answer["content"] == "":
- continue
- history_feedin.append(what_i_have_asked)
- history_feedin.append(what_gpt_answer)
- else:
- history_feedin[-1]['content'] = what_gpt_answer['content']
-
- # 开始接收回复
- response = f"[Local Message] 等待{model_name}响应中 ..."
- for response in _llm_handle.stream_chat(query=inputs, history=history_feedin, max_length=llm_kwargs['max_length'], top_p=llm_kwargs['top_p'], temperature=llm_kwargs['temperature']):
- chatbot[-1] = (inputs, response)
- yield from update_ui(chatbot=chatbot, history=history)
-
- # 总结输出
- if response == f"[Local Message] 等待{model_name}响应中 ...":
- response = f"[Local Message] {model_name}响应异常 ..."
- history.extend([inputs, response])
- yield from update_ui(chatbot=chatbot, history=history)
-
- return predict_no_ui_long_connection, predict
diff --git a/request_llms/moss b/request_llms/moss
deleted file mode 160000
index 4d905bcead53739d4395b145cae2be308b1df795..0000000000000000000000000000000000000000
--- a/request_llms/moss
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 4d905bcead53739d4395b145cae2be308b1df795
diff --git a/request_llms/queued_pipe.py b/request_llms/queued_pipe.py
deleted file mode 100644
index 1fc2e5bd53801898dbf0c98268de9e10c37bace9..0000000000000000000000000000000000000000
--- a/request_llms/queued_pipe.py
+++ /dev/null
@@ -1,24 +0,0 @@
-from multiprocessing import Pipe, Queue
-import time
-import threading
-
-class PipeSide(object):
- def __init__(self, q_2remote, q_2local) -> None:
- self.q_2remote = q_2remote
- self.q_2local = q_2local
-
- def recv(self):
- return self.q_2local.get()
-
- def send(self, buf):
- self.q_2remote.put(buf)
-
- def poll(self):
- return not self.q_2local.empty()
-
-def create_queue_pipe():
- q_p2c = Queue()
- q_c2p = Queue()
- pipe_c = PipeSide(q_2local=q_p2c, q_2remote=q_c2p)
- pipe_p = PipeSide(q_2local=q_c2p, q_2remote=q_p2c)
- return pipe_c, pipe_p
diff --git a/request_llms/requirements_qwen.txt b/request_llms/requirements_qwen.txt
deleted file mode 100644
index e9617907be68516ff85ab10097a5e52423664c8e..0000000000000000000000000000000000000000
--- a/request_llms/requirements_qwen.txt
+++ /dev/null
@@ -1 +0,0 @@
-dashscope
diff --git a/request_llms/requirements_qwen_local.txt b/request_llms/requirements_qwen_local.txt
deleted file mode 100644
index c4715f3f5e48ff41e828a86b69c2fa90c039fe14..0000000000000000000000000000000000000000
--- a/request_llms/requirements_qwen_local.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-modelscope
-transformers_stream_generator
-auto-gptq
-optimum
-urllib3<2
diff --git a/request_llms/requirements_slackclaude.txt b/request_llms/requirements_slackclaude.txt
deleted file mode 100644
index 7c8563efb570325a9145f1e9af972ee34b394904..0000000000000000000000000000000000000000
--- a/request_llms/requirements_slackclaude.txt
+++ /dev/null
@@ -1 +0,0 @@
-slack-sdk==3.21.3
diff --git a/requirements.txt b/requirements.txt
index 007c5a775de882e9e83ddceb1904d7be1b9fffac..0a9a4c83f90cd9796ff9e5c52b3512eeb4c19e68 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,28 +1,22 @@
-https://public.agent-matrix.com/publish/gradio-3.32.8-py3-none-any.whl
-gradio-client==0.8
-pypdf2==2.12.1
-zhipuai>=2
+pydantic==1.10.11
tiktoken>=0.3.3
requests[socks]
-pydantic==2.5.2
-protobuf==3.18
-transformers>=4.27.1
-scipdf_parser>=0.52
+transformers
python-markdown-math
-pymdown-extensions
-websocket-client
beautifulsoup4
prompt_toolkit
latex2mathml
python-docx
mdtex2html
anthropic
-pyautogen
colorama
Markdown
pygments
pymupdf
openai
-arxiv
numpy
+arxiv
rich
+pypdf2==2.12.1
+websocket-client
+scipdf_parser>=0.3
diff --git a/shared_utils/advanced_markdown_format.py b/shared_utils/advanced_markdown_format.py
deleted file mode 100644
index a015fd608914ed41982be743bc45ca9541d2166c..0000000000000000000000000000000000000000
--- a/shared_utils/advanced_markdown_format.py
+++ /dev/null
@@ -1,361 +0,0 @@
-import markdown
-import re
-import os
-import math
-from textwrap import dedent
-from functools import lru_cache
-from pymdownx.superfences import fence_code_format
-from latex2mathml.converter import convert as tex2mathml
-from shared_utils.config_loader import get_conf as get_conf
-from shared_utils.text_mask import apply_gpt_academic_string_mask
-
-markdown_extension_configs = {
- "mdx_math": {
- "enable_dollar_delimiter": True,
- "use_gitlab_delimiters": False,
- },
-}
-
-code_highlight_configs = {
- "pymdownx.superfences": {
- "css_class": "codehilite",
- "custom_fences": [
- {"name": "mermaid", "class": "mermaid", "format": fence_code_format}
- ],
- },
- "pymdownx.highlight": {
- "css_class": "codehilite",
- "guess_lang": True,
- # 'auto_title': True,
- # 'linenums': True
- },
-}
-
-code_highlight_configs_block_mermaid = {
- "pymdownx.superfences": {
- "css_class": "codehilite",
- # "custom_fences": [
- # {"name": "mermaid", "class": "mermaid", "format": fence_code_format}
- # ],
- },
- "pymdownx.highlight": {
- "css_class": "codehilite",
- "guess_lang": True,
- # 'auto_title': True,
- # 'linenums': True
- },
-}
-
-def tex2mathml_catch_exception(content, *args, **kwargs):
- try:
- content = tex2mathml(content, *args, **kwargs)
- except:
- content = content
- return content
-
-
-def replace_math_no_render(match):
- content = match.group(1)
- if "mode=display" in match.group(0):
- content = content.replace("\n", "")
- return f'$${content}$$'
- else:
- return f'${content}$'
-
-
-def replace_math_render(match):
- content = match.group(1)
- if "mode=display" in match.group(0):
- if "\\begin{aligned}" in content:
- content = content.replace("\\begin{aligned}", "\\begin{array}")
- content = content.replace("\\end{aligned}", "\\end{array}")
- content = content.replace("&", " ")
- content = tex2mathml_catch_exception(content, display="block")
- return content
- else:
- return tex2mathml_catch_exception(content)
-
-
-def markdown_bug_hunt(content):
- """
- 解决一个mdx_math的bug（单$包裹begin命令时多余\n", "")
- return content
-
-
-def is_equation(txt):
- """
- 判定是否为公式 | 测试1 写出洛伦兹定律，使用tex格式公式 测试2 给出柯西不等式，使用latex格式 测试3 写出麦克斯韦方程组
- """
- if "```" in txt and "```reference" not in txt:
- return False
- if "$" not in txt and "\\[" not in txt:
- return False
- mathpatterns = {
- r"(?^[ \t]*(?:~{3,}|`{3,}))[ ]* # opening fence
- ((\{(?P[^\}\n]*)\})| # (optional {attrs} or
- (\.?(?P[\w#.+-]*)[ ]*)? # optional (.)lang
- (hl_lines=(?P"|')(?P.*?)(?P=quot)[ ]*)?) # optional hl_lines)
- \n # newline (end of opening fence)
- (?P.*?)(?<=\n) # the code block
- (?P=fence)[ ]*$ # closing fence
- """
- ),
- re.MULTILINE | re.DOTALL | re.VERBOSE,
-)
-
-
-def get_line_range(re_match_obj, txt):
- start_pos, end_pos = re_match_obj.regs[0]
- num_newlines_before = txt[: start_pos + 1].count("\n")
- line_start = num_newlines_before
- line_end = num_newlines_before + txt[start_pos:end_pos].count("\n") + 1
- return line_start, line_end
-
-
-def fix_code_segment_indent(txt):
- lines = []
- change_any = False
- txt_tmp = txt
- while True:
- re_match_obj = FENCED_BLOCK_RE.search(txt_tmp)
- if not re_match_obj:
- break
- if len(lines) == 0:
- lines = txt.split("\n")
-
- # 清空 txt_tmp 对应的位置方便下次搜索
- start_pos, end_pos = re_match_obj.regs[0]
- txt_tmp = txt_tmp[:start_pos] + " " * (end_pos - start_pos) + txt_tmp[end_pos:]
- line_start, line_end = get_line_range(re_match_obj, txt)
-
- # 获取公共缩进
- shared_indent_cnt = 1e5
- for i in range(line_start, line_end):
- stripped_string = lines[i].lstrip()
- num_spaces = len(lines[i]) - len(stripped_string)
- if num_spaces < shared_indent_cnt:
- shared_indent_cnt = num_spaces
-
- # 修复缩进
- if (shared_indent_cnt < 1e5) and (shared_indent_cnt % 4) == 3:
- num_spaces_should_be = math.ceil(shared_indent_cnt / 4) * 4
- for i in range(line_start, line_end):
- add_n = num_spaces_should_be - shared_indent_cnt
- lines[i] = " " * add_n + lines[i]
- if not change_any: # 遇到第一个
- change_any = True
-
- if change_any:
- return "\n".join(lines)
- else:
- return txt
-
-
-@lru_cache(maxsize=128) # 使用 lru缓存 加快转换速度
-def markdown_convertion(txt):
- """
- 将Markdown格式的文本转换为HTML格式。如果包含数学公式，则先将公式转换为HTML格式。
- """
- pre = ''
- suf = "
"
- if txt.startswith(pre) and txt.endswith(suf):
- # print('警告，输入了已经经过转化的字符串，二次转化可能出问题')
- return txt # 已经被转化过，不需要再次转化
-
- find_equation_pattern = r'\n"""
-
- # 添加Live2D
- if ADD_WAIFU:
- for jsf in [
- "file=themes/waifu_plugin/jquery.min.js",
- "file=themes/waifu_plugin/jquery-ui.min.js",
- ]:
- js += f"""\n"""
- return js
\ No newline at end of file
diff --git a/themes/contrast.css b/themes/contrast.css
index 980a7e4fb6228723b8b83faf1892da9bad25298b..54a1b2b4480bde6ac17938fd73691b13f43d84ef 100644
--- a/themes/contrast.css
+++ b/themes/contrast.css
@@ -17,7 +17,7 @@
--button-primary-text-color-hover: #FFFFFF;
--button-secondary-text-color: #FFFFFF;
--button-secondary-text-color-hover: #FFFFFF;
-
+
--border-bottom-right-radius: 0px;
--border-bottom-left-radius: 0px;
@@ -51,8 +51,8 @@
--button-primary-border-color-hover: #3cff00;
--button-secondary-border-color: #3cff00;
--button-secondary-border-color-hover: #3cff00;
-
-
+
+
--body-background-fill: #000000;
--background-fill-primary: #000000;
--background-fill-secondary: #000000;
@@ -103,7 +103,7 @@
--button-primary-text-color-hover: #FFFFFF;
--button-secondary-text-color: #FFFFFF;
--button-secondary-text-color-hover: #FFFFFF;
-
+
--border-bottom-right-radius: 0px;
@@ -138,8 +138,8 @@
--button-primary-border-color-hover: #3cff00;
--button-secondary-border-color: #3cff00;
--button-secondary-border-color-hover: #3cff00;
-
-
+
+
--body-background-fill: #000000;
--background-fill-primary: #000000;
--background-fill-secondary: #000000;
@@ -198,7 +198,7 @@
}
/* 小按钮 */
-.sm {
+.sm.svelte-1ipelgc {
font-family: "Microsoft YaHei UI", "Helvetica", "Microsoft YaHei", "ui-sans-serif", "sans-serif", "system-ui";
--button-small-text-weight: 600;
--button-small-text-size: 16px;
@@ -208,7 +208,7 @@
border-top-left-radius: 0px;
}
-#plugin-panel .sm {
+#plugin-panel .sm.svelte-1ipelgc {
font-family: "Microsoft YaHei UI", "Helvetica", "Microsoft YaHei", "ui-sans-serif", "sans-serif", "system-ui";
--button-small-text-weight: 400;
--button-small-text-size: 14px;
@@ -479,3 +479,4 @@
.dark .codehilite .vi { color: #89DDFF } /* Name.Variable.Instance */
.dark .codehilite .vm { color: #82AAFF } /* Name.Variable.Magic */
.dark .codehilite .il { color: #F78C6C } /* Literal.Number.Integer.Long */
+
diff --git a/themes/contrast.py b/themes/contrast.py
index 1e98837755de43f0eb99b02dd4d344064ee600da..fd4ef0465bd97db3a4cb5c20ac749e364a640677 100644
--- a/themes/contrast.py
+++ b/themes/contrast.py
@@ -1,26 +1,16 @@
-import os
import gradio as gr
from toolbox import get_conf
-
-CODE_HIGHLIGHT, ADD_WAIFU, LAYOUT = get_conf("CODE_HIGHLIGHT", "ADD_WAIFU", "LAYOUT")
-theme_dir = os.path.dirname(__file__)
-
+CODE_HIGHLIGHT, ADD_WAIFU, LAYOUT = get_conf('CODE_HIGHLIGHT', 'ADD_WAIFU', 'LAYOUT')
def adjust_theme():
+
try:
color_er = gr.themes.utils.colors.fuchsia
set_theme = gr.themes.Default(
primary_hue=gr.themes.utils.colors.orange,
neutral_hue=gr.themes.utils.colors.gray,
- font=[
- "Helvetica",
- "Microsoft YaHei",
- "ui-sans-serif",
- "sans-serif",
- "system-ui",
- ],
- font_mono=["ui-monospace", "Consolas", "monospace"],
- )
+ font=["Helvetica", "Microsoft YaHei", "ui-sans-serif", "sans-serif", "system-ui"],
+ font_mono=["ui-monospace", "Consolas", "monospace"])
set_theme.set(
# Colors
input_background_fill_dark="*neutral_800",
@@ -67,29 +57,32 @@ def adjust_theme():
button_cancel_text_color_dark="white",
)
- from themes.common import get_common_html_javascript_code
- js = get_common_html_javascript_code()
-
- if not hasattr(gr, "RawTemplateResponse"):
- gr.RawTemplateResponse = gr.routes.templates.TemplateResponse
- gradio_original_template_fn = gr.RawTemplateResponse
-
+ if LAYOUT=="TOP-DOWN":
+ js = ""
+ else:
+ with open('themes/common.js', 'r', encoding='utf8') as f:
+ js = f""
+
+ # 添加一个萌萌的看板娘
+ if ADD_WAIFU:
+ js += """
+
+
+
+ """
+ gradio_original_template_fn = gr.routes.templates.TemplateResponse
def gradio_new_template_fn(*args, **kwargs):
res = gradio_original_template_fn(*args, **kwargs)
- res.body = res.body.replace(b"
GPT 学术优化 (GPT Academic)
-
+
-- 如果输出包含公式，会以tex形式和渲染形式同时显示，方便复制和阅读
+- 如果输出包含公式，会同时以tex形式和渲染形式显示，方便复制和阅读
-7. OpenAI图像生成
+7. 新增MOSS大语言模型支持
+
+
GPT الأكاديمي
-
-
-
-
-