Spaces:
Running
Running
Use pino for logs (#1086)
Browse files* Use pino for logs
* Use logger for info
* Add errorId to the logs
- package-lock.json +268 -3
- package.json +2 -0
- src/app.d.ts +5 -0
- src/hooks.server.ts +27 -1
- src/lib/assistantStats/refresh-assistants-counts.ts +3 -2
- src/lib/migrations/migrations.ts +9 -8
- src/lib/server/abortedGenerations.ts +2 -1
- src/lib/server/auth.ts +2 -1
- src/lib/server/database.ts +33 -32
- src/lib/server/embeddingEndpoints/hfApi/embeddingHfApi.ts +3 -2
- src/lib/server/embeddingEndpoints/tei/embeddingEndpoints.ts +2 -1
- src/lib/server/endpoints/cloudflare/endpointCloudflare.ts +3 -2
- src/lib/server/endpoints/langserve/endpointLangserve.ts +3 -2
- src/lib/server/endpoints/llamacpp/endpointLlamacpp.ts +3 -2
- src/lib/server/logger.ts +18 -0
- src/lib/server/models.ts +2 -1
- src/lib/server/preprocessMessages.ts +2 -1
- src/lib/server/summarize.ts +2 -1
- src/lib/server/websearch/searchSearxng.ts +2 -1
- src/routes/+error.svelte +5 -0
- src/routes/admin/export/+server.ts +7 -6
- src/routes/admin/stats/compute/+server.ts +6 -5
- src/routes/conversation/[id]/+server.ts +2 -1
- src/routes/settings/(nav)/assistants/[assistantId]/+page.server.ts +2 -1
package-lock.json
CHANGED
@@ -29,6 +29,8 @@
|
|
29 |
"nanoid": "^4.0.2",
|
30 |
"openid-client": "^5.4.2",
|
31 |
"parquetjs": "^0.11.2",
|
|
|
|
|
32 |
"postcss": "^8.4.31",
|
33 |
"saslprep": "^1.0.3",
|
34 |
"satori": "^0.10.11",
|
@@ -2451,7 +2453,6 @@
|
|
2451 |
"version": "3.0.0",
|
2452 |
"resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
|
2453 |
"integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",
|
2454 |
-
"optional": true,
|
2455 |
"dependencies": {
|
2456 |
"event-target-shim": "^5.0.0"
|
2457 |
},
|
@@ -2612,6 +2613,14 @@
|
|
2612 |
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
|
2613 |
"integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
|
2614 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2615 |
"node_modules/autoprefixer": {
|
2616 |
"version": "10.4.14",
|
2617 |
"resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.14.tgz",
|
@@ -3103,6 +3112,11 @@
|
|
3103 |
"simple-swizzle": "^0.2.2"
|
3104 |
}
|
3105 |
},
|
|
|
|
|
|
|
|
|
|
|
3106 |
"node_modules/combined-stream": {
|
3107 |
"version": "1.0.8",
|
3108 |
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
|
@@ -3313,6 +3327,14 @@
|
|
3313 |
"node": ">=6"
|
3314 |
}
|
3315 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
3316 |
"node_modules/debug": {
|
3317 |
"version": "4.3.4",
|
3318 |
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
|
@@ -3860,11 +3882,18 @@
|
|
3860 |
"version": "5.0.1",
|
3861 |
"resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
|
3862 |
"integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==",
|
3863 |
-
"optional": true,
|
3864 |
"engines": {
|
3865 |
"node": ">=6"
|
3866 |
}
|
3867 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
3868 |
"node_modules/execa": {
|
3869 |
"version": "5.1.1",
|
3870 |
"resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz",
|
@@ -3902,6 +3931,11 @@
|
|
3902 |
"integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==",
|
3903 |
"optional": true
|
3904 |
},
|
|
|
|
|
|
|
|
|
|
|
3905 |
"node_modules/fast-deep-equal": {
|
3906 |
"version": "3.1.3",
|
3907 |
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
|
@@ -3957,6 +3991,19 @@
|
|
3957 |
"integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==",
|
3958 |
"dev": true
|
3959 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
3960 |
"node_modules/fastq": {
|
3961 |
"version": "1.15.0",
|
3962 |
"resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz",
|
@@ -4430,6 +4477,11 @@
|
|
4430 |
"node": ">= 0.4"
|
4431 |
}
|
4432 |
},
|
|
|
|
|
|
|
|
|
|
|
4433 |
"node_modules/hex-rgb": {
|
4434 |
"version": "4.3.0",
|
4435 |
"resolved": "https://registry.npmjs.org/hex-rgb/-/hex-rgb-4.3.0.tgz",
|
@@ -4755,6 +4807,14 @@
|
|
4755 |
"url": "https://github.com/sponsors/panva"
|
4756 |
}
|
4757 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
4758 |
"node_modules/js-base64": {
|
4759 |
"version": "3.7.2",
|
4760 |
"resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.2.tgz",
|
@@ -5586,6 +5646,14 @@
|
|
5586 |
"node": "^10.13.0 || >=12.0.0"
|
5587 |
}
|
5588 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
5589 |
"node_modules/once": {
|
5590 |
"version": "1.4.0",
|
5591 |
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
|
@@ -5920,6 +5988,141 @@
|
|
5920 |
"node": ">=0.10.0"
|
5921 |
}
|
5922 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
5923 |
"node_modules/pirates": {
|
5924 |
"version": "4.0.5",
|
5925 |
"resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.5.tgz",
|
@@ -6313,6 +6516,19 @@
|
|
6313 |
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
|
6314 |
}
|
6315 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
6316 |
"node_modules/protobufjs": {
|
6317 |
"version": "6.11.4",
|
6318 |
"resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.4.tgz",
|
@@ -6421,6 +6637,11 @@
|
|
6421 |
"resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz",
|
6422 |
"integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag=="
|
6423 |
},
|
|
|
|
|
|
|
|
|
|
|
6424 |
"node_modules/rc": {
|
6425 |
"version": "1.2.8",
|
6426 |
"resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
|
@@ -6481,6 +6702,14 @@
|
|
6481 |
"node": ">=8.10.0"
|
6482 |
}
|
6483 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
6484 |
"node_modules/requires-port": {
|
6485 |
"version": "1.0.0",
|
6486 |
"resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
|
@@ -6609,6 +6838,14 @@
|
|
6609 |
}
|
6610 |
]
|
6611 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
6612 |
"node_modules/safer-buffer": {
|
6613 |
"version": "2.1.2",
|
6614 |
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
|
@@ -6688,6 +6925,11 @@
|
|
6688 |
"node": ">=v12.22.7"
|
6689 |
}
|
6690 |
},
|
|
|
|
|
|
|
|
|
|
|
6691 |
"node_modules/semver": {
|
6692 |
"version": "7.5.4",
|
6693 |
"resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
|
@@ -6924,6 +7166,14 @@
|
|
6924 |
"npm": ">= 3.0.0"
|
6925 |
}
|
6926 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
6927 |
"node_modules/sorcery": {
|
6928 |
"version": "0.11.0",
|
6929 |
"resolved": "https://registry.npmjs.org/sorcery/-/sorcery-0.11.0.tgz",
|
@@ -6963,6 +7213,14 @@
|
|
6963 |
"memory-pager": "^1.0.2"
|
6964 |
}
|
6965 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
6966 |
"node_modules/sprintf-js": {
|
6967 |
"version": "1.1.3",
|
6968 |
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz",
|
@@ -7039,7 +7297,6 @@
|
|
7039 |
"version": "3.1.1",
|
7040 |
"resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz",
|
7041 |
"integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==",
|
7042 |
-
"dev": true,
|
7043 |
"engines": {
|
7044 |
"node": ">=8"
|
7045 |
},
|
@@ -7471,6 +7728,14 @@
|
|
7471 |
"node": ">=0.8"
|
7472 |
}
|
7473 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
7474 |
"node_modules/thrift": {
|
7475 |
"version": "0.11.0",
|
7476 |
"resolved": "https://registry.npmjs.org/thrift/-/thrift-0.11.0.tgz",
|
|
|
29 |
"nanoid": "^4.0.2",
|
30 |
"openid-client": "^5.4.2",
|
31 |
"parquetjs": "^0.11.2",
|
32 |
+
"pino": "^9.0.0",
|
33 |
+
"pino-pretty": "^11.0.0",
|
34 |
"postcss": "^8.4.31",
|
35 |
"saslprep": "^1.0.3",
|
36 |
"satori": "^0.10.11",
|
|
|
2453 |
"version": "3.0.0",
|
2454 |
"resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
|
2455 |
"integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",
|
|
|
2456 |
"dependencies": {
|
2457 |
"event-target-shim": "^5.0.0"
|
2458 |
},
|
|
|
2613 |
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
|
2614 |
"integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
|
2615 |
},
|
2616 |
+
"node_modules/atomic-sleep": {
|
2617 |
+
"version": "1.0.0",
|
2618 |
+
"resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz",
|
2619 |
+
"integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==",
|
2620 |
+
"engines": {
|
2621 |
+
"node": ">=8.0.0"
|
2622 |
+
}
|
2623 |
+
},
|
2624 |
"node_modules/autoprefixer": {
|
2625 |
"version": "10.4.14",
|
2626 |
"resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.14.tgz",
|
|
|
3112 |
"simple-swizzle": "^0.2.2"
|
3113 |
}
|
3114 |
},
|
3115 |
+
"node_modules/colorette": {
|
3116 |
+
"version": "2.0.20",
|
3117 |
+
"resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz",
|
3118 |
+
"integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w=="
|
3119 |
+
},
|
3120 |
"node_modules/combined-stream": {
|
3121 |
"version": "1.0.8",
|
3122 |
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
|
|
|
3327 |
"node": ">=6"
|
3328 |
}
|
3329 |
},
|
3330 |
+
"node_modules/dateformat": {
|
3331 |
+
"version": "4.6.3",
|
3332 |
+
"resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz",
|
3333 |
+
"integrity": "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==",
|
3334 |
+
"engines": {
|
3335 |
+
"node": "*"
|
3336 |
+
}
|
3337 |
+
},
|
3338 |
"node_modules/debug": {
|
3339 |
"version": "4.3.4",
|
3340 |
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
|
|
|
3882 |
"version": "5.0.1",
|
3883 |
"resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
|
3884 |
"integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==",
|
|
|
3885 |
"engines": {
|
3886 |
"node": ">=6"
|
3887 |
}
|
3888 |
},
|
3889 |
+
"node_modules/events": {
|
3890 |
+
"version": "3.3.0",
|
3891 |
+
"resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
|
3892 |
+
"integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==",
|
3893 |
+
"engines": {
|
3894 |
+
"node": ">=0.8.x"
|
3895 |
+
}
|
3896 |
+
},
|
3897 |
"node_modules/execa": {
|
3898 |
"version": "5.1.1",
|
3899 |
"resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz",
|
|
|
3931 |
"integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==",
|
3932 |
"optional": true
|
3933 |
},
|
3934 |
+
"node_modules/fast-copy": {
|
3935 |
+
"version": "3.0.2",
|
3936 |
+
"resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-3.0.2.tgz",
|
3937 |
+
"integrity": "sha512-dl0O9Vhju8IrcLndv2eU4ldt1ftXMqqfgN4H1cpmGV7P6jeB9FwpN9a2c8DPGE1Ys88rNUJVYDHq73CGAGOPfQ=="
|
3938 |
+
},
|
3939 |
"node_modules/fast-deep-equal": {
|
3940 |
"version": "3.1.3",
|
3941 |
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
|
|
|
3991 |
"integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==",
|
3992 |
"dev": true
|
3993 |
},
|
3994 |
+
"node_modules/fast-redact": {
|
3995 |
+
"version": "3.5.0",
|
3996 |
+
"resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.5.0.tgz",
|
3997 |
+
"integrity": "sha512-dwsoQlS7h9hMeYUq1W++23NDcBLV4KqONnITDV9DjfS3q1SgDGVrBdvvTLUotWtPSD7asWDV9/CmsZPy8Hf70A==",
|
3998 |
+
"engines": {
|
3999 |
+
"node": ">=6"
|
4000 |
+
}
|
4001 |
+
},
|
4002 |
+
"node_modules/fast-safe-stringify": {
|
4003 |
+
"version": "2.1.1",
|
4004 |
+
"resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz",
|
4005 |
+
"integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA=="
|
4006 |
+
},
|
4007 |
"node_modules/fastq": {
|
4008 |
"version": "1.15.0",
|
4009 |
"resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz",
|
|
|
4477 |
"node": ">= 0.4"
|
4478 |
}
|
4479 |
},
|
4480 |
+
"node_modules/help-me": {
|
4481 |
+
"version": "5.0.0",
|
4482 |
+
"resolved": "https://registry.npmjs.org/help-me/-/help-me-5.0.0.tgz",
|
4483 |
+
"integrity": "sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg=="
|
4484 |
+
},
|
4485 |
"node_modules/hex-rgb": {
|
4486 |
"version": "4.3.0",
|
4487 |
"resolved": "https://registry.npmjs.org/hex-rgb/-/hex-rgb-4.3.0.tgz",
|
|
|
4807 |
"url": "https://github.com/sponsors/panva"
|
4808 |
}
|
4809 |
},
|
4810 |
+
"node_modules/joycon": {
|
4811 |
+
"version": "3.1.1",
|
4812 |
+
"resolved": "https://registry.npmjs.org/joycon/-/joycon-3.1.1.tgz",
|
4813 |
+
"integrity": "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==",
|
4814 |
+
"engines": {
|
4815 |
+
"node": ">=10"
|
4816 |
+
}
|
4817 |
+
},
|
4818 |
"node_modules/js-base64": {
|
4819 |
"version": "3.7.2",
|
4820 |
"resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.2.tgz",
|
|
|
5646 |
"node": "^10.13.0 || >=12.0.0"
|
5647 |
}
|
5648 |
},
|
5649 |
+
"node_modules/on-exit-leak-free": {
|
5650 |
+
"version": "2.1.2",
|
5651 |
+
"resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz",
|
5652 |
+
"integrity": "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==",
|
5653 |
+
"engines": {
|
5654 |
+
"node": ">=14.0.0"
|
5655 |
+
}
|
5656 |
+
},
|
5657 |
"node_modules/once": {
|
5658 |
"version": "1.4.0",
|
5659 |
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
|
|
|
5988 |
"node": ">=0.10.0"
|
5989 |
}
|
5990 |
},
|
5991 |
+
"node_modules/pino": {
|
5992 |
+
"version": "9.0.0",
|
5993 |
+
"resolved": "https://registry.npmjs.org/pino/-/pino-9.0.0.tgz",
|
5994 |
+
"integrity": "sha512-uI1ThkzTShNSwvsUM6b4ND8ANzWURk9zTELMztFkmnCQeR/4wkomJ+echHee5GMWGovoSfjwdeu80DsFIt7mbA==",
|
5995 |
+
"dependencies": {
|
5996 |
+
"atomic-sleep": "^1.0.0",
|
5997 |
+
"fast-redact": "^3.1.1",
|
5998 |
+
"on-exit-leak-free": "^2.1.0",
|
5999 |
+
"pino-abstract-transport": "^1.2.0",
|
6000 |
+
"pino-std-serializers": "^6.0.0",
|
6001 |
+
"process-warning": "^3.0.0",
|
6002 |
+
"quick-format-unescaped": "^4.0.3",
|
6003 |
+
"real-require": "^0.2.0",
|
6004 |
+
"safe-stable-stringify": "^2.3.1",
|
6005 |
+
"sonic-boom": "^3.7.0",
|
6006 |
+
"thread-stream": "^2.6.0"
|
6007 |
+
},
|
6008 |
+
"bin": {
|
6009 |
+
"pino": "bin.js"
|
6010 |
+
}
|
6011 |
+
},
|
6012 |
+
"node_modules/pino-abstract-transport": {
|
6013 |
+
"version": "1.2.0",
|
6014 |
+
"resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-1.2.0.tgz",
|
6015 |
+
"integrity": "sha512-Guhh8EZfPCfH+PMXAb6rKOjGQEoy0xlAIn+irODG5kgfYV+BQ0rGYYWTIel3P5mmyXqkYkPmdIkywsn6QKUR1Q==",
|
6016 |
+
"dependencies": {
|
6017 |
+
"readable-stream": "^4.0.0",
|
6018 |
+
"split2": "^4.0.0"
|
6019 |
+
}
|
6020 |
+
},
|
6021 |
+
"node_modules/pino-abstract-transport/node_modules/buffer": {
|
6022 |
+
"version": "6.0.3",
|
6023 |
+
"resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz",
|
6024 |
+
"integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==",
|
6025 |
+
"funding": [
|
6026 |
+
{
|
6027 |
+
"type": "github",
|
6028 |
+
"url": "https://github.com/sponsors/feross"
|
6029 |
+
},
|
6030 |
+
{
|
6031 |
+
"type": "patreon",
|
6032 |
+
"url": "https://www.patreon.com/feross"
|
6033 |
+
},
|
6034 |
+
{
|
6035 |
+
"type": "consulting",
|
6036 |
+
"url": "https://feross.org/support"
|
6037 |
+
}
|
6038 |
+
],
|
6039 |
+
"dependencies": {
|
6040 |
+
"base64-js": "^1.3.1",
|
6041 |
+
"ieee754": "^1.2.1"
|
6042 |
+
}
|
6043 |
+
},
|
6044 |
+
"node_modules/pino-abstract-transport/node_modules/readable-stream": {
|
6045 |
+
"version": "4.5.2",
|
6046 |
+
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz",
|
6047 |
+
"integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==",
|
6048 |
+
"dependencies": {
|
6049 |
+
"abort-controller": "^3.0.0",
|
6050 |
+
"buffer": "^6.0.3",
|
6051 |
+
"events": "^3.3.0",
|
6052 |
+
"process": "^0.11.10",
|
6053 |
+
"string_decoder": "^1.3.0"
|
6054 |
+
},
|
6055 |
+
"engines": {
|
6056 |
+
"node": "^12.22.0 || ^14.17.0 || >=16.0.0"
|
6057 |
+
}
|
6058 |
+
},
|
6059 |
+
"node_modules/pino-pretty": {
|
6060 |
+
"version": "11.0.0",
|
6061 |
+
"resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-11.0.0.tgz",
|
6062 |
+
"integrity": "sha512-YFJZqw59mHIY72wBnBs7XhLGG6qpJMa4pEQTRgEPEbjIYbng2LXEZZF1DoyDg9CfejEy8uZCyzpcBXXG0oOCwQ==",
|
6063 |
+
"dependencies": {
|
6064 |
+
"colorette": "^2.0.7",
|
6065 |
+
"dateformat": "^4.6.3",
|
6066 |
+
"fast-copy": "^3.0.0",
|
6067 |
+
"fast-safe-stringify": "^2.1.1",
|
6068 |
+
"help-me": "^5.0.0",
|
6069 |
+
"joycon": "^3.1.1",
|
6070 |
+
"minimist": "^1.2.6",
|
6071 |
+
"on-exit-leak-free": "^2.1.0",
|
6072 |
+
"pino-abstract-transport": "^1.0.0",
|
6073 |
+
"pump": "^3.0.0",
|
6074 |
+
"readable-stream": "^4.0.0",
|
6075 |
+
"secure-json-parse": "^2.4.0",
|
6076 |
+
"sonic-boom": "^3.0.0",
|
6077 |
+
"strip-json-comments": "^3.1.1"
|
6078 |
+
},
|
6079 |
+
"bin": {
|
6080 |
+
"pino-pretty": "bin.js"
|
6081 |
+
}
|
6082 |
+
},
|
6083 |
+
"node_modules/pino-pretty/node_modules/buffer": {
|
6084 |
+
"version": "6.0.3",
|
6085 |
+
"resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz",
|
6086 |
+
"integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==",
|
6087 |
+
"funding": [
|
6088 |
+
{
|
6089 |
+
"type": "github",
|
6090 |
+
"url": "https://github.com/sponsors/feross"
|
6091 |
+
},
|
6092 |
+
{
|
6093 |
+
"type": "patreon",
|
6094 |
+
"url": "https://www.patreon.com/feross"
|
6095 |
+
},
|
6096 |
+
{
|
6097 |
+
"type": "consulting",
|
6098 |
+
"url": "https://feross.org/support"
|
6099 |
+
}
|
6100 |
+
],
|
6101 |
+
"dependencies": {
|
6102 |
+
"base64-js": "^1.3.1",
|
6103 |
+
"ieee754": "^1.2.1"
|
6104 |
+
}
|
6105 |
+
},
|
6106 |
+
"node_modules/pino-pretty/node_modules/readable-stream": {
|
6107 |
+
"version": "4.5.2",
|
6108 |
+
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz",
|
6109 |
+
"integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==",
|
6110 |
+
"dependencies": {
|
6111 |
+
"abort-controller": "^3.0.0",
|
6112 |
+
"buffer": "^6.0.3",
|
6113 |
+
"events": "^3.3.0",
|
6114 |
+
"process": "^0.11.10",
|
6115 |
+
"string_decoder": "^1.3.0"
|
6116 |
+
},
|
6117 |
+
"engines": {
|
6118 |
+
"node": "^12.22.0 || ^14.17.0 || >=16.0.0"
|
6119 |
+
}
|
6120 |
+
},
|
6121 |
+
"node_modules/pino-std-serializers": {
|
6122 |
+
"version": "6.2.2",
|
6123 |
+
"resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-6.2.2.tgz",
|
6124 |
+
"integrity": "sha512-cHjPPsE+vhj/tnhCy/wiMh3M3z3h/j15zHQX+S9GkTBgqJuTuJzYJ4gUyACLhDaJ7kk9ba9iRDmbH2tJU03OiA=="
|
6125 |
+
},
|
6126 |
"node_modules/pirates": {
|
6127 |
"version": "4.0.5",
|
6128 |
"resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.5.tgz",
|
|
|
6516 |
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
|
6517 |
}
|
6518 |
},
|
6519 |
+
"node_modules/process": {
|
6520 |
+
"version": "0.11.10",
|
6521 |
+
"resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz",
|
6522 |
+
"integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==",
|
6523 |
+
"engines": {
|
6524 |
+
"node": ">= 0.6.0"
|
6525 |
+
}
|
6526 |
+
},
|
6527 |
+
"node_modules/process-warning": {
|
6528 |
+
"version": "3.0.0",
|
6529 |
+
"resolved": "https://registry.npmjs.org/process-warning/-/process-warning-3.0.0.tgz",
|
6530 |
+
"integrity": "sha512-mqn0kFRl0EoqhnL0GQ0veqFHyIN1yig9RHh/InzORTUiZHFRAur+aMtRkELNwGs9aNwKS6tg/An4NYBPGwvtzQ=="
|
6531 |
+
},
|
6532 |
"node_modules/protobufjs": {
|
6533 |
"version": "6.11.4",
|
6534 |
"resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.4.tgz",
|
|
|
6637 |
"resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz",
|
6638 |
"integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag=="
|
6639 |
},
|
6640 |
+
"node_modules/quick-format-unescaped": {
|
6641 |
+
"version": "4.0.4",
|
6642 |
+
"resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz",
|
6643 |
+
"integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg=="
|
6644 |
+
},
|
6645 |
"node_modules/rc": {
|
6646 |
"version": "1.2.8",
|
6647 |
"resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
|
|
|
6702 |
"node": ">=8.10.0"
|
6703 |
}
|
6704 |
},
|
6705 |
+
"node_modules/real-require": {
|
6706 |
+
"version": "0.2.0",
|
6707 |
+
"resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz",
|
6708 |
+
"integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==",
|
6709 |
+
"engines": {
|
6710 |
+
"node": ">= 12.13.0"
|
6711 |
+
}
|
6712 |
+
},
|
6713 |
"node_modules/requires-port": {
|
6714 |
"version": "1.0.0",
|
6715 |
"resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
|
|
|
6838 |
}
|
6839 |
]
|
6840 |
},
|
6841 |
+
"node_modules/safe-stable-stringify": {
|
6842 |
+
"version": "2.4.3",
|
6843 |
+
"resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz",
|
6844 |
+
"integrity": "sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==",
|
6845 |
+
"engines": {
|
6846 |
+
"node": ">=10"
|
6847 |
+
}
|
6848 |
+
},
|
6849 |
"node_modules/safer-buffer": {
|
6850 |
"version": "2.1.2",
|
6851 |
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
|
|
|
6925 |
"node": ">=v12.22.7"
|
6926 |
}
|
6927 |
},
|
6928 |
+
"node_modules/secure-json-parse": {
|
6929 |
+
"version": "2.7.0",
|
6930 |
+
"resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz",
|
6931 |
+
"integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw=="
|
6932 |
+
},
|
6933 |
"node_modules/semver": {
|
6934 |
"version": "7.5.4",
|
6935 |
"resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
|
|
|
7166 |
"npm": ">= 3.0.0"
|
7167 |
}
|
7168 |
},
|
7169 |
+
"node_modules/sonic-boom": {
|
7170 |
+
"version": "3.8.1",
|
7171 |
+
"resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-3.8.1.tgz",
|
7172 |
+
"integrity": "sha512-y4Z8LCDBuum+PBP3lSV7RHrXscqksve/bi0as7mhwVnBW+/wUqKT/2Kb7um8yqcFy0duYbbPxzt89Zy2nOCaxg==",
|
7173 |
+
"dependencies": {
|
7174 |
+
"atomic-sleep": "^1.0.0"
|
7175 |
+
}
|
7176 |
+
},
|
7177 |
"node_modules/sorcery": {
|
7178 |
"version": "0.11.0",
|
7179 |
"resolved": "https://registry.npmjs.org/sorcery/-/sorcery-0.11.0.tgz",
|
|
|
7213 |
"memory-pager": "^1.0.2"
|
7214 |
}
|
7215 |
},
|
7216 |
+
"node_modules/split2": {
|
7217 |
+
"version": "4.2.0",
|
7218 |
+
"resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz",
|
7219 |
+
"integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==",
|
7220 |
+
"engines": {
|
7221 |
+
"node": ">= 10.x"
|
7222 |
+
}
|
7223 |
+
},
|
7224 |
"node_modules/sprintf-js": {
|
7225 |
"version": "1.1.3",
|
7226 |
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz",
|
|
|
7297 |
"version": "3.1.1",
|
7298 |
"resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz",
|
7299 |
"integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==",
|
|
|
7300 |
"engines": {
|
7301 |
"node": ">=8"
|
7302 |
},
|
|
|
7728 |
"node": ">=0.8"
|
7729 |
}
|
7730 |
},
|
7731 |
+
"node_modules/thread-stream": {
|
7732 |
+
"version": "2.7.0",
|
7733 |
+
"resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-2.7.0.tgz",
|
7734 |
+
"integrity": "sha512-qQiRWsU/wvNolI6tbbCKd9iKaTnCXsTwVxhhKM6nctPdujTyztjlbUkUTUymidWcMnZ5pWR0ej4a0tjsW021vw==",
|
7735 |
+
"dependencies": {
|
7736 |
+
"real-require": "^0.2.0"
|
7737 |
+
}
|
7738 |
+
},
|
7739 |
"node_modules/thrift": {
|
7740 |
"version": "0.11.0",
|
7741 |
"resolved": "https://registry.npmjs.org/thrift/-/thrift-0.11.0.tgz",
|
package.json
CHANGED
@@ -69,6 +69,8 @@
|
|
69 |
"nanoid": "^4.0.2",
|
70 |
"openid-client": "^5.4.2",
|
71 |
"parquetjs": "^0.11.2",
|
|
|
|
|
72 |
"postcss": "^8.4.31",
|
73 |
"saslprep": "^1.0.3",
|
74 |
"satori": "^0.10.11",
|
|
|
69 |
"nanoid": "^4.0.2",
|
70 |
"openid-client": "^5.4.2",
|
71 |
"parquetjs": "^0.11.2",
|
72 |
+
"pino": "^9.0.0",
|
73 |
+
"pino-pretty": "^11.0.0",
|
74 |
"postcss": "^8.4.31",
|
75 |
"saslprep": "^1.0.3",
|
76 |
"satori": "^0.10.11",
|
src/app.d.ts
CHANGED
@@ -12,6 +12,11 @@ declare global {
|
|
12 |
sessionId: string;
|
13 |
user?: User;
|
14 |
}
|
|
|
|
|
|
|
|
|
|
|
15 |
// interface PageData {}
|
16 |
// interface Platform {}
|
17 |
}
|
|
|
12 |
sessionId: string;
|
13 |
user?: User;
|
14 |
}
|
15 |
+
|
16 |
+
interface Error {
|
17 |
+
message: string;
|
18 |
+
errorId?: ReturnType<typeof crypto.randomUUID>;
|
19 |
+
}
|
20 |
// interface PageData {}
|
21 |
// interface Platform {}
|
22 |
}
|
src/hooks.server.ts
CHANGED
@@ -6,7 +6,7 @@ import {
|
|
6 |
MESSAGES_BEFORE_LOGIN,
|
7 |
PARQUET_EXPORT_SECRET,
|
8 |
} from "$env/static/private";
|
9 |
-
import type { Handle } from "@sveltejs/kit";
|
10 |
import {
|
11 |
PUBLIC_GOOGLE_ANALYTICS_ID,
|
12 |
PUBLIC_ORIGIN,
|
@@ -21,6 +21,7 @@ import { addWeeks } from "date-fns";
|
|
21 |
import { checkAndRunMigrations } from "$lib/migrations/migrations";
|
22 |
import { building } from "$app/environment";
|
23 |
import { refreshAssistantsCounts } from "$lib/assistantStats/refresh-assistants-counts";
|
|
|
24 |
|
25 |
if (!building) {
|
26 |
await checkAndRunMigrations();
|
@@ -29,6 +30,31 @@ if (!building) {
|
|
29 |
}
|
30 |
}
|
31 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
32 |
export const handle: Handle = async ({ event, resolve }) => {
|
33 |
if (event.url.pathname.startsWith(`${base}/api/`) && EXPOSE_API !== "true") {
|
34 |
return new Response("API is disabled", { status: 403 });
|
|
|
6 |
MESSAGES_BEFORE_LOGIN,
|
7 |
PARQUET_EXPORT_SECRET,
|
8 |
} from "$env/static/private";
|
9 |
+
import type { Handle, HandleServerError } from "@sveltejs/kit";
|
10 |
import {
|
11 |
PUBLIC_GOOGLE_ANALYTICS_ID,
|
12 |
PUBLIC_ORIGIN,
|
|
|
21 |
import { checkAndRunMigrations } from "$lib/migrations/migrations";
|
22 |
import { building } from "$app/environment";
|
23 |
import { refreshAssistantsCounts } from "$lib/assistantStats/refresh-assistants-counts";
|
24 |
+
import { logger } from "$lib/server/logger";
|
25 |
|
26 |
if (!building) {
|
27 |
await checkAndRunMigrations();
|
|
|
30 |
}
|
31 |
}
|
32 |
|
33 |
+
export const handleError: HandleServerError = async ({ error, event }) => {
|
34 |
+
// handle 404
|
35 |
+
if (event.route.id === null) {
|
36 |
+
return {
|
37 |
+
message: `Page ${event.url.pathname} not found`,
|
38 |
+
};
|
39 |
+
}
|
40 |
+
|
41 |
+
const errorId = crypto.randomUUID();
|
42 |
+
|
43 |
+
logger.error({
|
44 |
+
locals: event.locals,
|
45 |
+
url: event.request.url,
|
46 |
+
params: event.params,
|
47 |
+
request: event.request,
|
48 |
+
error,
|
49 |
+
errorId,
|
50 |
+
});
|
51 |
+
|
52 |
+
return {
|
53 |
+
message: "An error occurred",
|
54 |
+
errorId,
|
55 |
+
};
|
56 |
+
};
|
57 |
+
|
58 |
export const handle: Handle = async ({ event, resolve }) => {
|
59 |
if (event.url.pathname.startsWith(`${base}/api/`) && EXPOSE_API !== "true") {
|
60 |
return new Response("API is disabled", { status: 403 });
|
src/lib/assistantStats/refresh-assistants-counts.ts
CHANGED
@@ -2,6 +2,7 @@ import { client, collections } from "$lib/server/database";
|
|
2 |
import { acquireLock, refreshLock } from "$lib/migrations/lock";
|
3 |
import type { ObjectId } from "mongodb";
|
4 |
import { subDays } from "date-fns";
|
|
|
5 |
|
6 |
const LOCK_KEY = "assistants.count";
|
7 |
|
@@ -53,8 +54,8 @@ async function refreshAssistantsCountsHelper() {
|
|
53 |
})
|
54 |
);
|
55 |
} catch (e) {
|
56 |
-
|
57 |
-
|
58 |
}
|
59 |
}
|
60 |
|
|
|
2 |
import { acquireLock, refreshLock } from "$lib/migrations/lock";
|
3 |
import type { ObjectId } from "mongodb";
|
4 |
import { subDays } from "date-fns";
|
5 |
+
import { logger } from "$lib/server/logger";
|
6 |
|
7 |
const LOCK_KEY = "assistants.count";
|
8 |
|
|
|
54 |
})
|
55 |
);
|
56 |
} catch (e) {
|
57 |
+
logger.error("Refresh assistants counter failed!");
|
58 |
+
logger.error(e);
|
59 |
}
|
60 |
}
|
61 |
|
src/lib/migrations/migrations.ts
CHANGED
@@ -2,6 +2,7 @@ import { client, collections } from "$lib/server/database";
|
|
2 |
import { migrations } from "./routines";
|
3 |
import { acquireLock, releaseLock, isDBLocked, refreshLock } from "./lock";
|
4 |
import { isHuggingChat } from "$lib/utils/isHuggingChat";
|
|
|
5 |
|
6 |
const LOCK_KEY = "migrations";
|
7 |
|
@@ -14,7 +15,7 @@ export async function checkAndRunMigrations() {
|
|
14 |
// check if all migrations have already been run
|
15 |
const migrationResults = await collections.migrationResults.find().toArray();
|
16 |
|
17 |
-
|
18 |
|
19 |
// connect to the database
|
20 |
const connectedClient = await client.connect();
|
@@ -23,7 +24,7 @@ export async function checkAndRunMigrations() {
|
|
23 |
|
24 |
if (!lockId) {
|
25 |
// another instance already has the lock, so we exit early
|
26 |
-
|
27 |
"[MIGRATIONS] Another instance already has the lock. Waiting for DB to be unlocked."
|
28 |
);
|
29 |
|
@@ -50,21 +51,21 @@ export async function checkAndRunMigrations() {
|
|
50 |
|
51 |
// check if the migration has already been applied
|
52 |
if (!shouldRun) {
|
53 |
-
|
54 |
} else {
|
55 |
// check the modifiers to see if some cases match
|
56 |
if (
|
57 |
(migration.runForHuggingChat === "only" && !isHuggingChat) ||
|
58 |
(migration.runForHuggingChat === "never" && isHuggingChat)
|
59 |
) {
|
60 |
-
|
61 |
`[MIGRATIONS] "${migration.name}" should not be applied for this run. Skipping...`
|
62 |
);
|
63 |
continue;
|
64 |
}
|
65 |
|
66 |
// otherwise all is good and we can run the migration
|
67 |
-
|
68 |
`[MIGRATIONS] "${migration.name}" ${
|
69 |
migration.runEveryTime ? "should run every time" : "not applied yet"
|
70 |
}. Applying...`
|
@@ -89,8 +90,8 @@ export async function checkAndRunMigrations() {
|
|
89 |
result = await migration.up(connectedClient);
|
90 |
});
|
91 |
} catch (e) {
|
92 |
-
|
93 |
-
|
94 |
} finally {
|
95 |
await session.endSession();
|
96 |
}
|
@@ -108,7 +109,7 @@ export async function checkAndRunMigrations() {
|
|
108 |
}
|
109 |
}
|
110 |
|
111 |
-
|
112 |
|
113 |
clearInterval(refreshInterval);
|
114 |
await releaseLock(LOCK_KEY, lockId);
|
|
|
2 |
import { migrations } from "./routines";
|
3 |
import { acquireLock, releaseLock, isDBLocked, refreshLock } from "./lock";
|
4 |
import { isHuggingChat } from "$lib/utils/isHuggingChat";
|
5 |
+
import { logger } from "$lib/server/logger";
|
6 |
|
7 |
const LOCK_KEY = "migrations";
|
8 |
|
|
|
15 |
// check if all migrations have already been run
|
16 |
const migrationResults = await collections.migrationResults.find().toArray();
|
17 |
|
18 |
+
logger.info("[MIGRATIONS] Begin check...");
|
19 |
|
20 |
// connect to the database
|
21 |
const connectedClient = await client.connect();
|
|
|
24 |
|
25 |
if (!lockId) {
|
26 |
// another instance already has the lock, so we exit early
|
27 |
+
logger.info(
|
28 |
"[MIGRATIONS] Another instance already has the lock. Waiting for DB to be unlocked."
|
29 |
);
|
30 |
|
|
|
51 |
|
52 |
// check if the migration has already been applied
|
53 |
if (!shouldRun) {
|
54 |
+
logger.info(`[MIGRATIONS] "${migration.name}" already applied. Skipping...`);
|
55 |
} else {
|
56 |
// check the modifiers to see if some cases match
|
57 |
if (
|
58 |
(migration.runForHuggingChat === "only" && !isHuggingChat) ||
|
59 |
(migration.runForHuggingChat === "never" && isHuggingChat)
|
60 |
) {
|
61 |
+
logger.info(
|
62 |
`[MIGRATIONS] "${migration.name}" should not be applied for this run. Skipping...`
|
63 |
);
|
64 |
continue;
|
65 |
}
|
66 |
|
67 |
// otherwise all is good and we can run the migration
|
68 |
+
logger.info(
|
69 |
`[MIGRATIONS] "${migration.name}" ${
|
70 |
migration.runEveryTime ? "should run every time" : "not applied yet"
|
71 |
}. Applying...`
|
|
|
90 |
result = await migration.up(connectedClient);
|
91 |
});
|
92 |
} catch (e) {
|
93 |
+
logger.info(`[MIGRATIONS] "${migration.name}" failed!`);
|
94 |
+
logger.error(e);
|
95 |
} finally {
|
96 |
await session.endSession();
|
97 |
}
|
|
|
109 |
}
|
110 |
}
|
111 |
|
112 |
+
logger.info("[MIGRATIONS] All migrations applied. Releasing lock");
|
113 |
|
114 |
clearInterval(refreshInterval);
|
115 |
await releaseLock(LOCK_KEY, lockId);
|
src/lib/server/abortedGenerations.ts
CHANGED
@@ -2,6 +2,7 @@
|
|
2 |
|
3 |
import { setTimeout } from "node:timers/promises";
|
4 |
import { collections } from "./database";
|
|
|
5 |
|
6 |
let closed = false;
|
7 |
process.on("SIGINT", () => {
|
@@ -21,7 +22,7 @@ async function maintainAbortedGenerations() {
|
|
21 |
aborts.map(({ conversationId, createdAt }) => [conversationId.toString(), createdAt])
|
22 |
);
|
23 |
} catch (err) {
|
24 |
-
|
25 |
}
|
26 |
}
|
27 |
}
|
|
|
2 |
|
3 |
import { setTimeout } from "node:timers/promises";
|
4 |
import { collections } from "./database";
|
5 |
+
import { logger } from "$lib/server/logger";
|
6 |
|
7 |
let closed = false;
|
8 |
process.on("SIGINT", () => {
|
|
|
22 |
aborts.map(({ conversationId, createdAt }) => [conversationId.toString(), createdAt])
|
23 |
);
|
24 |
} catch (err) {
|
25 |
+
logger.error(err);
|
26 |
}
|
27 |
}
|
28 |
}
|
src/lib/server/auth.ts
CHANGED
@@ -18,6 +18,7 @@ import { dev } from "$app/environment";
|
|
18 |
import type { Cookies } from "@sveltejs/kit";
|
19 |
import { collections } from "./database";
|
20 |
import JSON5 from "json5";
|
|
|
21 |
|
22 |
export interface OIDCSettings {
|
23 |
redirectURI: string;
|
@@ -151,7 +152,7 @@ export async function validateAndParseCsrfToken(
|
|
151 |
return { redirectUrl: data.redirectUrl };
|
152 |
}
|
153 |
} catch (e) {
|
154 |
-
|
155 |
}
|
156 |
return null;
|
157 |
}
|
|
|
18 |
import type { Cookies } from "@sveltejs/kit";
|
19 |
import { collections } from "./database";
|
20 |
import JSON5 from "json5";
|
21 |
+
import { logger } from "$lib/server/logger";
|
22 |
|
23 |
export interface OIDCSettings {
|
24 |
redirectURI: string;
|
|
|
152 |
return { redirectUrl: data.redirectUrl };
|
153 |
}
|
154 |
} catch (e) {
|
155 |
+
logger.error(e);
|
156 |
}
|
157 |
return null;
|
158 |
}
|
src/lib/server/database.ts
CHANGED
@@ -13,6 +13,7 @@ import type { ConversationStats } from "$lib/types/ConversationStats";
|
|
13 |
import type { MigrationResult } from "$lib/types/MigrationResult";
|
14 |
import type { Semaphore } from "$lib/types/Semaphore";
|
15 |
import type { AssistantStats } from "$lib/types/AssistantStats";
|
|
|
16 |
|
17 |
if (!MONGODB_URL) {
|
18 |
throw new Error(
|
@@ -25,7 +26,7 @@ const client = new MongoClient(MONGODB_URL, {
|
|
25 |
directConnection: MONGODB_DIRECT_CONNECTION === "true",
|
26 |
});
|
27 |
|
28 |
-
export const connectPromise = client.connect().catch(
|
29 |
|
30 |
export function getCollections(mongoClient: MongoClient) {
|
31 |
const db = mongoClient.db(MONGODB_DB_NAME + (import.meta.env.MODE === "test" ? "-test" : ""));
|
@@ -89,25 +90,25 @@ client.on("open", () => {
|
|
89 |
{ sessionId: 1, updatedAt: -1 },
|
90 |
{ partialFilterExpression: { sessionId: { $exists: true } } }
|
91 |
)
|
92 |
-
.catch(
|
93 |
conversations
|
94 |
.createIndex(
|
95 |
{ userId: 1, updatedAt: -1 },
|
96 |
{ partialFilterExpression: { userId: { $exists: true } } }
|
97 |
)
|
98 |
-
.catch(
|
99 |
conversations
|
100 |
.createIndex(
|
101 |
{ "message.id": 1, "message.ancestors": 1 },
|
102 |
{ partialFilterExpression: { userId: { $exists: true } } }
|
103 |
)
|
104 |
-
.catch(
|
105 |
// To do stats on conversations
|
106 |
-
conversations.createIndex({ updatedAt: 1 }).catch(
|
107 |
// Not strictly necessary, could use _id, but more convenient. Also for stats
|
108 |
-
conversations.createIndex({ createdAt: 1 }).catch(
|
109 |
// To do stats on conversation messages
|
110 |
-
conversations.createIndex({ "messages.createdAt": 1 }, { sparse: true }).catch(
|
111 |
// Unique index for stats
|
112 |
conversationStats
|
113 |
.createIndex(
|
@@ -120,7 +121,7 @@ client.on("open", () => {
|
|
120 |
},
|
121 |
{ unique: true }
|
122 |
)
|
123 |
-
.catch(
|
124 |
// Allow easy check of last computed stat for given type/dateField
|
125 |
conversationStats
|
126 |
.createIndex({
|
@@ -128,34 +129,34 @@ client.on("open", () => {
|
|
128 |
"date.field": 1,
|
129 |
"date.at": 1,
|
130 |
})
|
131 |
-
.catch(
|
132 |
-
abortedGenerations.createIndex({ updatedAt: 1 }, { expireAfterSeconds: 30 }).catch(
|
133 |
-
abortedGenerations.createIndex({ conversationId: 1 }, { unique: true }).catch(
|
134 |
-
sharedConversations.createIndex({ hash: 1 }, { unique: true }).catch(
|
135 |
-
settings.createIndex({ sessionId: 1 }, { unique: true, sparse: true }).catch(
|
136 |
-
settings.createIndex({ userId: 1 }, { unique: true, sparse: true }).catch(
|
137 |
-
settings.createIndex({ assistants: 1 }).catch(
|
138 |
-
users.createIndex({ hfUserId: 1 }, { unique: true }).catch(
|
139 |
-
users.createIndex({ sessionId: 1 }, { unique: true, sparse: true }).catch(
|
140 |
// No unicity because due to renames & outdated info from oauth provider, there may be the same username on different users
|
141 |
-
users.createIndex({ username: 1 }).catch(
|
142 |
-
messageEvents.createIndex({ createdAt: 1 }, { expireAfterSeconds: 60 }).catch(
|
143 |
-
sessions.createIndex({ expiresAt: 1 }, { expireAfterSeconds: 0 }).catch(
|
144 |
-
sessions.createIndex({ sessionId: 1 }, { unique: true }).catch(
|
145 |
-
assistants.createIndex({ createdById: 1, userCount: -1 }).catch(
|
146 |
-
assistants.createIndex({ userCount: 1 }).catch(
|
147 |
-
assistants.createIndex({ featured: 1, userCount: -1 }).catch(
|
148 |
-
assistants.createIndex({ modelId: 1, userCount: -1 }).catch(
|
149 |
-
assistants.createIndex({ searchTokens: 1 }).catch(
|
150 |
-
assistants.createIndex({ last24HoursCount: 1 }).catch(
|
151 |
assistantStats
|
152 |
// Order of keys is important for the queries
|
153 |
.createIndex({ "date.span": 1, "date.at": 1, assistantId: 1 }, { unique: true })
|
154 |
-
.catch(
|
155 |
-
reports.createIndex({ assistantId: 1 }).catch(
|
156 |
-
reports.createIndex({ createdBy: 1, assistantId: 1 }).catch(
|
157 |
|
158 |
// Unique index for semaphore and migration results
|
159 |
-
semaphores.createIndex({ key: 1 }, { unique: true }).catch(
|
160 |
-
semaphores.createIndex({ createdAt: 1 }, { expireAfterSeconds: 60 }).catch(
|
161 |
});
|
|
|
13 |
import type { MigrationResult } from "$lib/types/MigrationResult";
|
14 |
import type { Semaphore } from "$lib/types/Semaphore";
|
15 |
import type { AssistantStats } from "$lib/types/AssistantStats";
|
16 |
+
import { logger } from "$lib/server/logger";
|
17 |
|
18 |
if (!MONGODB_URL) {
|
19 |
throw new Error(
|
|
|
26 |
directConnection: MONGODB_DIRECT_CONNECTION === "true",
|
27 |
});
|
28 |
|
29 |
+
export const connectPromise = client.connect().catch(logger.error);
|
30 |
|
31 |
export function getCollections(mongoClient: MongoClient) {
|
32 |
const db = mongoClient.db(MONGODB_DB_NAME + (import.meta.env.MODE === "test" ? "-test" : ""));
|
|
|
90 |
{ sessionId: 1, updatedAt: -1 },
|
91 |
{ partialFilterExpression: { sessionId: { $exists: true } } }
|
92 |
)
|
93 |
+
.catch(logger.error);
|
94 |
conversations
|
95 |
.createIndex(
|
96 |
{ userId: 1, updatedAt: -1 },
|
97 |
{ partialFilterExpression: { userId: { $exists: true } } }
|
98 |
)
|
99 |
+
.catch(logger.error);
|
100 |
conversations
|
101 |
.createIndex(
|
102 |
{ "message.id": 1, "message.ancestors": 1 },
|
103 |
{ partialFilterExpression: { userId: { $exists: true } } }
|
104 |
)
|
105 |
+
.catch(logger.error);
|
106 |
// To do stats on conversations
|
107 |
+
conversations.createIndex({ updatedAt: 1 }).catch(logger.error);
|
108 |
// Not strictly necessary, could use _id, but more convenient. Also for stats
|
109 |
+
conversations.createIndex({ createdAt: 1 }).catch(logger.error);
|
110 |
// To do stats on conversation messages
|
111 |
+
conversations.createIndex({ "messages.createdAt": 1 }, { sparse: true }).catch(logger.error);
|
112 |
// Unique index for stats
|
113 |
conversationStats
|
114 |
.createIndex(
|
|
|
121 |
},
|
122 |
{ unique: true }
|
123 |
)
|
124 |
+
.catch(logger.error);
|
125 |
// Allow easy check of last computed stat for given type/dateField
|
126 |
conversationStats
|
127 |
.createIndex({
|
|
|
129 |
"date.field": 1,
|
130 |
"date.at": 1,
|
131 |
})
|
132 |
+
.catch(logger.error);
|
133 |
+
abortedGenerations.createIndex({ updatedAt: 1 }, { expireAfterSeconds: 30 }).catch(logger.error);
|
134 |
+
abortedGenerations.createIndex({ conversationId: 1 }, { unique: true }).catch(logger.error);
|
135 |
+
sharedConversations.createIndex({ hash: 1 }, { unique: true }).catch(logger.error);
|
136 |
+
settings.createIndex({ sessionId: 1 }, { unique: true, sparse: true }).catch(logger.error);
|
137 |
+
settings.createIndex({ userId: 1 }, { unique: true, sparse: true }).catch(logger.error);
|
138 |
+
settings.createIndex({ assistants: 1 }).catch(logger.error);
|
139 |
+
users.createIndex({ hfUserId: 1 }, { unique: true }).catch(logger.error);
|
140 |
+
users.createIndex({ sessionId: 1 }, { unique: true, sparse: true }).catch(logger.error);
|
141 |
// No unicity because due to renames & outdated info from oauth provider, there may be the same username on different users
|
142 |
+
users.createIndex({ username: 1 }).catch(logger.error);
|
143 |
+
messageEvents.createIndex({ createdAt: 1 }, { expireAfterSeconds: 60 }).catch(logger.error);
|
144 |
+
sessions.createIndex({ expiresAt: 1 }, { expireAfterSeconds: 0 }).catch(logger.error);
|
145 |
+
sessions.createIndex({ sessionId: 1 }, { unique: true }).catch(logger.error);
|
146 |
+
assistants.createIndex({ createdById: 1, userCount: -1 }).catch(logger.error);
|
147 |
+
assistants.createIndex({ userCount: 1 }).catch(logger.error);
|
148 |
+
assistants.createIndex({ featured: 1, userCount: -1 }).catch(logger.error);
|
149 |
+
assistants.createIndex({ modelId: 1, userCount: -1 }).catch(logger.error);
|
150 |
+
assistants.createIndex({ searchTokens: 1 }).catch(logger.error);
|
151 |
+
assistants.createIndex({ last24HoursCount: 1 }).catch(logger.error);
|
152 |
assistantStats
|
153 |
// Order of keys is important for the queries
|
154 |
.createIndex({ "date.span": 1, "date.at": 1, assistantId: 1 }, { unique: true })
|
155 |
+
.catch(logger.error);
|
156 |
+
reports.createIndex({ assistantId: 1 }).catch(logger.error);
|
157 |
+
reports.createIndex({ createdBy: 1, assistantId: 1 }).catch(logger.error);
|
158 |
|
159 |
// Unique index for semaphore and migration results
|
160 |
+
semaphores.createIndex({ key: 1 }, { unique: true }).catch(logger.error);
|
161 |
+
semaphores.createIndex({ createdAt: 1 }, { expireAfterSeconds: 60 }).catch(logger.error);
|
162 |
});
|
src/lib/server/embeddingEndpoints/hfApi/embeddingHfApi.ts
CHANGED
@@ -2,6 +2,7 @@ import { z } from "zod";
|
|
2 |
import type { EmbeddingEndpoint, Embedding } from "../embeddingEndpoints";
|
3 |
import { chunk } from "$lib/utils/chunk";
|
4 |
import { HF_TOKEN } from "$env/static/private";
|
|
|
5 |
|
6 |
export const embeddingEndpointHfApiSchema = z.object({
|
7 |
weight: z.number().int().positive().default(1),
|
@@ -35,8 +36,8 @@ export async function embeddingEndpointHfApi(
|
|
35 |
});
|
36 |
|
37 |
if (!response.ok) {
|
38 |
-
|
39 |
-
|
40 |
return [];
|
41 |
}
|
42 |
|
|
|
2 |
import type { EmbeddingEndpoint, Embedding } from "../embeddingEndpoints";
|
3 |
import { chunk } from "$lib/utils/chunk";
|
4 |
import { HF_TOKEN } from "$env/static/private";
|
5 |
+
import { logger } from "$lib/server/logger";
|
6 |
|
7 |
export const embeddingEndpointHfApiSchema = z.object({
|
8 |
weight: z.number().int().positive().default(1),
|
|
|
36 |
});
|
37 |
|
38 |
if (!response.ok) {
|
39 |
+
logger.error(await response.text());
|
40 |
+
logger.error("Failed to get embeddings from Hugging Face API", response);
|
41 |
return [];
|
42 |
}
|
43 |
|
src/lib/server/embeddingEndpoints/tei/embeddingEndpoints.ts
CHANGED
@@ -2,6 +2,7 @@ import { z } from "zod";
|
|
2 |
import type { EmbeddingEndpoint, Embedding } from "../embeddingEndpoints";
|
3 |
import { chunk } from "$lib/utils/chunk";
|
4 |
import { HF_TOKEN } from "$env/static/private";
|
|
|
5 |
|
6 |
export const embeddingEndpointTeiParametersSchema = z.object({
|
7 |
weight: z.number().int().positive().default(1),
|
@@ -29,7 +30,7 @@ const getModelInfoByUrl = async (url: string, authorization?: string) => {
|
|
29 |
const json = await response.json();
|
30 |
return { max_client_batch_size: 32, max_batch_tokens: 16384, ...json };
|
31 |
} catch {
|
32 |
-
|
33 |
return { max_client_batch_size: 32, max_batch_tokens: 16384 };
|
34 |
}
|
35 |
};
|
|
|
2 |
import type { EmbeddingEndpoint, Embedding } from "../embeddingEndpoints";
|
3 |
import { chunk } from "$lib/utils/chunk";
|
4 |
import { HF_TOKEN } from "$env/static/private";
|
5 |
+
import { logger } from "$lib/server/logger";
|
6 |
|
7 |
export const embeddingEndpointTeiParametersSchema = z.object({
|
8 |
weight: z.number().int().positive().default(1),
|
|
|
30 |
const json = await response.json();
|
31 |
return { max_client_batch_size: 32, max_batch_tokens: 16384, ...json };
|
32 |
} catch {
|
33 |
+
logger.debug("Could not get info from TEI embedding endpoint. Using defaults.");
|
34 |
return { max_client_batch_size: 32, max_batch_tokens: 16384 };
|
35 |
}
|
36 |
};
|
src/lib/server/endpoints/cloudflare/endpointCloudflare.ts
CHANGED
@@ -2,6 +2,7 @@ import { z } from "zod";
|
|
2 |
import type { Endpoint } from "../endpoints";
|
3 |
import type { TextGenerationStreamOutput } from "@huggingface/inference";
|
4 |
import { CLOUDFLARE_ACCOUNT_ID, CLOUDFLARE_API_TOKEN } from "$env/static/private";
|
|
|
5 |
|
6 |
export const endpointCloudflareParametersSchema = z.object({
|
7 |
weight: z.number().int().positive().default(1),
|
@@ -104,8 +105,8 @@ export async function endpointCloudflare(
|
|
104 |
try {
|
105 |
data = JSON.parse(jsonString);
|
106 |
} catch (e) {
|
107 |
-
|
108 |
-
|
109 |
continue; // Skip this iteration and try the next chunk
|
110 |
}
|
111 |
|
|
|
2 |
import type { Endpoint } from "../endpoints";
|
3 |
import type { TextGenerationStreamOutput } from "@huggingface/inference";
|
4 |
import { CLOUDFLARE_ACCOUNT_ID, CLOUDFLARE_API_TOKEN } from "$env/static/private";
|
5 |
+
import { logger } from "$lib/server/logger";
|
6 |
|
7 |
export const endpointCloudflareParametersSchema = z.object({
|
8 |
weight: z.number().int().positive().default(1),
|
|
|
105 |
try {
|
106 |
data = JSON.parse(jsonString);
|
107 |
} catch (e) {
|
108 |
+
logger.error("Failed to parse JSON", e);
|
109 |
+
logger.error("Problematic JSON string:", jsonString);
|
110 |
continue; // Skip this iteration and try the next chunk
|
111 |
}
|
112 |
|
src/lib/server/endpoints/langserve/endpointLangserve.ts
CHANGED
@@ -2,6 +2,7 @@ import { buildPrompt } from "$lib/buildPrompt";
|
|
2 |
import { z } from "zod";
|
3 |
import type { Endpoint } from "../endpoints";
|
4 |
import type { TextGenerationStreamOutput } from "@huggingface/inference";
|
|
|
5 |
|
6 |
export const endpointLangserveParametersSchema = z.object({
|
7 |
weight: z.number().int().positive().default(1),
|
@@ -99,8 +100,8 @@ export function endpointLangserve(
|
|
99 |
try {
|
100 |
data = JSON.parse(jsonString);
|
101 |
} catch (e) {
|
102 |
-
|
103 |
-
|
104 |
continue; // Skip this iteration and try the next chunk
|
105 |
}
|
106 |
// Assuming content within data is a plain string
|
|
|
2 |
import { z } from "zod";
|
3 |
import type { Endpoint } from "../endpoints";
|
4 |
import type { TextGenerationStreamOutput } from "@huggingface/inference";
|
5 |
+
import { logger } from "$lib/server/logger";
|
6 |
|
7 |
export const endpointLangserveParametersSchema = z.object({
|
8 |
weight: z.number().int().positive().default(1),
|
|
|
100 |
try {
|
101 |
data = JSON.parse(jsonString);
|
102 |
} catch (e) {
|
103 |
+
logger.error("Failed to parse JSON", e);
|
104 |
+
logger.error("Problematic JSON string:", jsonString);
|
105 |
continue; // Skip this iteration and try the next chunk
|
106 |
}
|
107 |
// Assuming content within data is a plain string
|
src/lib/server/endpoints/llamacpp/endpointLlamacpp.ts
CHANGED
@@ -3,6 +3,7 @@ import { buildPrompt } from "$lib/buildPrompt";
|
|
3 |
import type { TextGenerationStreamOutput } from "@huggingface/inference";
|
4 |
import type { Endpoint } from "../endpoints";
|
5 |
import { z } from "zod";
|
|
|
6 |
|
7 |
export const endpointLlamacppParametersSchema = z.object({
|
8 |
weight: z.number().int().positive().default(1),
|
@@ -93,8 +94,8 @@ export function endpointLlamacpp(
|
|
93 |
try {
|
94 |
data = JSON.parse(jsonString);
|
95 |
} catch (e) {
|
96 |
-
|
97 |
-
|
98 |
continue; // Skip this iteration and try the next chunk
|
99 |
}
|
100 |
|
|
|
3 |
import type { TextGenerationStreamOutput } from "@huggingface/inference";
|
4 |
import type { Endpoint } from "../endpoints";
|
5 |
import { z } from "zod";
|
6 |
+
import { logger } from "$lib/server/logger";
|
7 |
|
8 |
export const endpointLlamacppParametersSchema = z.object({
|
9 |
weight: z.number().int().positive().default(1),
|
|
|
94 |
try {
|
95 |
data = JSON.parse(jsonString);
|
96 |
} catch (e) {
|
97 |
+
logger.error("Failed to parse JSON", e);
|
98 |
+
logger.error("Problematic JSON string:", jsonString);
|
99 |
continue; // Skip this iteration and try the next chunk
|
100 |
}
|
101 |
|
src/lib/server/logger.ts
ADDED
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import pino from "pino";
|
2 |
+
import { dev } from "$app/environment";
|
3 |
+
|
4 |
+
let options: pino.LoggerOptions = {};
|
5 |
+
|
6 |
+
if (dev) {
|
7 |
+
options = {
|
8 |
+
level: "debug",
|
9 |
+
transport: {
|
10 |
+
target: "pino-pretty",
|
11 |
+
options: {
|
12 |
+
colorize: true,
|
13 |
+
},
|
14 |
+
},
|
15 |
+
};
|
16 |
+
}
|
17 |
+
|
18 |
+
export const logger = pino(options);
|
src/lib/server/models.ts
CHANGED
@@ -18,6 +18,7 @@ import type { PreTrainedTokenizer } from "@xenova/transformers";
|
|
18 |
|
19 |
import JSON5 from "json5";
|
20 |
import { getTokenizer } from "$lib/utils/getTokenizer";
|
|
|
21 |
|
22 |
type Optional<T, K extends keyof T> = Pick<Partial<T>, K> & Omit<T, K>;
|
23 |
|
@@ -91,7 +92,7 @@ async function getChatPromptRender(
|
|
91 |
try {
|
92 |
tokenizer = await getTokenizer(m.tokenizer);
|
93 |
} catch (e) {
|
94 |
-
|
95 |
"Failed to load tokenizer for model " +
|
96 |
m.name +
|
97 |
" consider setting chatPromptTemplate manually or making sure the model is available on the hub. Error: " +
|
|
|
18 |
|
19 |
import JSON5 from "json5";
|
20 |
import { getTokenizer } from "$lib/utils/getTokenizer";
|
21 |
+
import { logger } from "$lib/server/logger";
|
22 |
|
23 |
type Optional<T, K extends keyof T> = Pick<Partial<T>, K> & Omit<T, K>;
|
24 |
|
|
|
92 |
try {
|
93 |
tokenizer = await getTokenizer(m.tokenizer);
|
94 |
} catch (e) {
|
95 |
+
logger.error(
|
96 |
"Failed to load tokenizer for model " +
|
97 |
m.name +
|
98 |
" consider setting chatPromptTemplate manually or making sure the model is available on the hub. Error: " +
|
src/lib/server/preprocessMessages.ts
CHANGED
@@ -2,6 +2,7 @@ import type { Conversation } from "$lib/types/Conversation";
|
|
2 |
import type { Message } from "$lib/types/Message";
|
3 |
import { format } from "date-fns";
|
4 |
import { downloadFile } from "./files/downloadFile";
|
|
|
5 |
|
6 |
export async function preprocessMessages(
|
7 |
messages: Message[],
|
@@ -44,7 +45,7 @@ Answer the question: ${lastQuestion}`;
|
|
44 |
const b64 = image.toString("base64");
|
45 |
return `![](data:${mime};base64,${b64})})`;
|
46 |
} catch (e) {
|
47 |
-
|
48 |
}
|
49 |
})
|
50 |
);
|
|
|
2 |
import type { Message } from "$lib/types/Message";
|
3 |
import { format } from "date-fns";
|
4 |
import { downloadFile } from "./files/downloadFile";
|
5 |
+
import { logger } from "$lib/server/logger";
|
6 |
|
7 |
export async function preprocessMessages(
|
8 |
messages: Message[],
|
|
|
45 |
const b64 = image.toString("base64");
|
46 |
return `![](data:${mime};base64,${b64})})`;
|
47 |
} catch (e) {
|
48 |
+
logger.error(e);
|
49 |
}
|
50 |
})
|
51 |
);
|
src/lib/server/summarize.ts
CHANGED
@@ -1,6 +1,7 @@
|
|
1 |
import { LLM_SUMMERIZATION } from "$env/static/private";
|
2 |
import { generateFromDefaultEndpoint } from "$lib/server/generateFromDefaultEndpoint";
|
3 |
import type { Message } from "$lib/types/Message";
|
|
|
4 |
|
5 |
export async function summarize(prompt: string) {
|
6 |
if (!LLM_SUMMERIZATION) {
|
@@ -41,7 +42,7 @@ export async function summarize(prompt: string) {
|
|
41 |
return summary;
|
42 |
})
|
43 |
.catch((e) => {
|
44 |
-
|
45 |
return null;
|
46 |
});
|
47 |
}
|
|
|
1 |
import { LLM_SUMMERIZATION } from "$env/static/private";
|
2 |
import { generateFromDefaultEndpoint } from "$lib/server/generateFromDefaultEndpoint";
|
3 |
import type { Message } from "$lib/types/Message";
|
4 |
+
import { logger } from "$lib/server/logger";
|
5 |
|
6 |
export async function summarize(prompt: string) {
|
7 |
if (!LLM_SUMMERIZATION) {
|
|
|
42 |
return summary;
|
43 |
})
|
44 |
.catch((e) => {
|
45 |
+
logger.error(e);
|
46 |
return null;
|
47 |
});
|
48 |
}
|
src/lib/server/websearch/searchSearxng.ts
CHANGED
@@ -1,4 +1,5 @@
|
|
1 |
import { SEARXNG_QUERY_URL } from "$env/static/private";
|
|
|
2 |
|
3 |
export async function searchSearxng(query: string) {
|
4 |
const abortController = new AbortController();
|
@@ -18,7 +19,7 @@ export async function searchSearxng(query: string) {
|
|
18 |
})
|
19 |
.then((response) => response.json() as Promise<{ results: { url: string }[] }>)
|
20 |
.catch((error) => {
|
21 |
-
|
22 |
throw new Error("Failed to fetch or parse JSON");
|
23 |
});
|
24 |
|
|
|
1 |
import { SEARXNG_QUERY_URL } from "$env/static/private";
|
2 |
+
import { logger } from "$lib/server/logger";
|
3 |
|
4 |
export async function searchSearxng(query: string) {
|
5 |
const abortController = new AbortController();
|
|
|
19 |
})
|
20 |
.then((response) => response.json() as Promise<{ results: { url: string }[] }>)
|
21 |
.catch((error) => {
|
22 |
+
logger.error("Failed to fetch or parse JSON", error);
|
23 |
throw new Error("Failed to fetch or parse JSON");
|
24 |
});
|
25 |
|
src/routes/+error.svelte
CHANGED
@@ -11,5 +11,10 @@
|
|
11 |
<h1 class="mb-2 text-5xl font-semibold">{$page.status}</h1>
|
12 |
<div class="-mx-8 my-2 h-px bg-gray-200 dark:bg-gray-700" />
|
13 |
<h2 class="max-w-sm text-lg">{$page.error?.message}</h2>
|
|
|
|
|
|
|
|
|
|
|
14 |
</div>
|
15 |
</div>
|
|
|
11 |
<h1 class="mb-2 text-5xl font-semibold">{$page.status}</h1>
|
12 |
<div class="-mx-8 my-2 h-px bg-gray-200 dark:bg-gray-700" />
|
13 |
<h2 class="max-w-sm text-lg">{$page.error?.message}</h2>
|
14 |
+
{#if $page.error?.errorId}
|
15 |
+
<div class="-mx-8 my-2 h-px bg-gray-200 dark:bg-gray-700" />
|
16 |
+
<pre class="max-w-sm whitespace-pre-wrap text-left font-mono text-xs">{$page.error
|
17 |
+
.errorId}</pre>
|
18 |
+
{/if}
|
19 |
</div>
|
20 |
</div>
|
src/routes/admin/export/+server.ts
CHANGED
@@ -7,6 +7,7 @@ import { unlink } from "node:fs/promises";
|
|
7 |
import { uploadFile } from "@huggingface/hub";
|
8 |
import parquet from "parquetjs";
|
9 |
import { z } from "zod";
|
|
|
10 |
|
11 |
// Triger like this:
|
12 |
// curl -X POST "http://localhost:5173/chat/admin/export" -H "Authorization: Bearer <ADMIN_API_SECRET>" -H "Content-Type: application/json" -d '{"model": "OpenAssistant/oasst-sft-6-llama-30b-xor"}'
|
@@ -41,7 +42,7 @@ export async function POST({ request }) {
|
|
41 |
const writer = await parquet.ParquetWriter.openFile(schema, fileName);
|
42 |
|
43 |
let count = 0;
|
44 |
-
|
45 |
|
46 |
for await (const conversation of collections.settings.aggregate<{
|
47 |
title: string;
|
@@ -88,11 +89,11 @@ export async function POST({ request }) {
|
|
88 |
++count;
|
89 |
|
90 |
if (count % 1_000 === 0) {
|
91 |
-
|
92 |
}
|
93 |
}
|
94 |
|
95 |
-
|
96 |
|
97 |
for await (const conversation of collections.settings.aggregate<{
|
98 |
title: string;
|
@@ -133,13 +134,13 @@ export async function POST({ request }) {
|
|
133 |
++count;
|
134 |
|
135 |
if (count % 1_000 === 0) {
|
136 |
-
|
137 |
}
|
138 |
}
|
139 |
|
140 |
await writer.close();
|
141 |
|
142 |
-
|
143 |
|
144 |
await uploadFile({
|
145 |
file: pathToFileURL(fileName) as URL,
|
@@ -150,7 +151,7 @@ export async function POST({ request }) {
|
|
150 |
},
|
151 |
});
|
152 |
|
153 |
-
|
154 |
|
155 |
await unlink(fileName);
|
156 |
|
|
|
7 |
import { uploadFile } from "@huggingface/hub";
|
8 |
import parquet from "parquetjs";
|
9 |
import { z } from "zod";
|
10 |
+
import { logger } from "$lib/server/logger.js";
|
11 |
|
12 |
// Triger like this:
|
13 |
// curl -X POST "http://localhost:5173/chat/admin/export" -H "Authorization: Bearer <ADMIN_API_SECRET>" -H "Content-Type: application/json" -d '{"model": "OpenAssistant/oasst-sft-6-llama-30b-xor"}'
|
|
|
42 |
const writer = await parquet.ParquetWriter.openFile(schema, fileName);
|
43 |
|
44 |
let count = 0;
|
45 |
+
logger.info("Exporting conversations for model", model);
|
46 |
|
47 |
for await (const conversation of collections.settings.aggregate<{
|
48 |
title: string;
|
|
|
89 |
++count;
|
90 |
|
91 |
if (count % 1_000 === 0) {
|
92 |
+
logger.info("Exported", count, "conversations");
|
93 |
}
|
94 |
}
|
95 |
|
96 |
+
logger.info("exporting convos with userId");
|
97 |
|
98 |
for await (const conversation of collections.settings.aggregate<{
|
99 |
title: string;
|
|
|
134 |
++count;
|
135 |
|
136 |
if (count % 1_000 === 0) {
|
137 |
+
logger.info("Exported", count, "conversations");
|
138 |
}
|
139 |
}
|
140 |
|
141 |
await writer.close();
|
142 |
|
143 |
+
logger.info("Uploading", fileName, "to Hugging Face Hub");
|
144 |
|
145 |
await uploadFile({
|
146 |
file: pathToFileURL(fileName) as URL,
|
|
|
151 |
},
|
152 |
});
|
153 |
|
154 |
+
logger.info("Upload done");
|
155 |
|
156 |
await unlink(fileName);
|
157 |
|
src/routes/admin/stats/compute/+server.ts
CHANGED
@@ -1,15 +1,16 @@
|
|
1 |
import { json } from "@sveltejs/kit";
|
2 |
import type { ConversationStats } from "$lib/types/ConversationStats";
|
3 |
import { CONVERSATION_STATS_COLLECTION, collections } from "$lib/server/database.js";
|
|
|
4 |
|
5 |
// Triger like this:
|
6 |
// curl -X POST "http://localhost:5173/chat/admin/stats/compute" -H "Authorization: Bearer <ADMIN_API_SECRET>"
|
7 |
|
8 |
export async function POST() {
|
9 |
for (const span of ["day", "week", "month"] as const) {
|
10 |
-
computeStats({ dateField: "updatedAt", type: "conversation", span }).catch(
|
11 |
-
computeStats({ dateField: "createdAt", type: "conversation", span }).catch(
|
12 |
-
computeStats({ dateField: "createdAt", type: "message", span }).catch(
|
13 |
}
|
14 |
|
15 |
return json({}, { status: 202 });
|
@@ -29,7 +30,7 @@ async function computeStats(params: {
|
|
29 |
// In those cases we need to compute the stats from before the last month as everything is one aggregation
|
30 |
const minDate = lastComputed ? lastComputed.date.at : new Date(0);
|
31 |
|
32 |
-
|
33 |
|
34 |
const dateField = params.type === "message" ? "messages." + params.dateField : params.dateField;
|
35 |
|
@@ -213,5 +214,5 @@ async function computeStats(params: {
|
|
213 |
|
214 |
await collections.conversations.aggregate(pipeline, { allowDiskUse: true }).next();
|
215 |
|
216 |
-
|
217 |
}
|
|
|
1 |
import { json } from "@sveltejs/kit";
|
2 |
import type { ConversationStats } from "$lib/types/ConversationStats";
|
3 |
import { CONVERSATION_STATS_COLLECTION, collections } from "$lib/server/database.js";
|
4 |
+
import { logger } from "$lib/server/logger";
|
5 |
|
6 |
// Triger like this:
|
7 |
// curl -X POST "http://localhost:5173/chat/admin/stats/compute" -H "Authorization: Bearer <ADMIN_API_SECRET>"
|
8 |
|
9 |
export async function POST() {
|
10 |
for (const span of ["day", "week", "month"] as const) {
|
11 |
+
computeStats({ dateField: "updatedAt", type: "conversation", span }).catch(logger.error);
|
12 |
+
computeStats({ dateField: "createdAt", type: "conversation", span }).catch(logger.error);
|
13 |
+
computeStats({ dateField: "createdAt", type: "message", span }).catch(logger.error);
|
14 |
}
|
15 |
|
16 |
return json({}, { status: 202 });
|
|
|
30 |
// In those cases we need to compute the stats from before the last month as everything is one aggregation
|
31 |
const minDate = lastComputed ? lastComputed.date.at : new Date(0);
|
32 |
|
33 |
+
logger.info("Computing stats for", params.type, params.span, params.dateField, "from", minDate);
|
34 |
|
35 |
const dateField = params.type === "message" ? "messages." + params.dateField : params.dateField;
|
36 |
|
|
|
214 |
|
215 |
await collections.conversations.aggregate(pipeline, { allowDiskUse: true }).next();
|
216 |
|
217 |
+
logger.info("Computed stats for", params.type, params.span, params.dateField);
|
218 |
}
|
src/routes/conversation/[id]/+server.ts
CHANGED
@@ -23,6 +23,7 @@ import { addSibling } from "$lib/utils/tree/addSibling.js";
|
|
23 |
import { preprocessMessages } from "$lib/server/preprocessMessages.js";
|
24 |
import { usageLimits } from "$lib/server/usageLimits";
|
25 |
import { isURLLocal } from "$lib/server/isURLLocal.js";
|
|
|
26 |
|
27 |
export async function POST({ request, locals, params, getClientAddress }) {
|
28 |
const id = z.string().parse(params.id);
|
@@ -334,7 +335,7 @@ export async function POST({ request, locals, params, getClientAddress }) {
|
|
334 |
}
|
335 |
);
|
336 |
} catch (e) {
|
337 |
-
|
338 |
}
|
339 |
}
|
340 |
})();
|
|
|
23 |
import { preprocessMessages } from "$lib/server/preprocessMessages.js";
|
24 |
import { usageLimits } from "$lib/server/usageLimits";
|
25 |
import { isURLLocal } from "$lib/server/isURLLocal.js";
|
26 |
+
import { logger } from "$lib/server/logger.js";
|
27 |
|
28 |
export async function POST({ request, locals, params, getClientAddress }) {
|
29 |
const id = z.string().parse(params.id);
|
|
|
335 |
}
|
336 |
);
|
337 |
} catch (e) {
|
338 |
+
logger.error(e);
|
339 |
}
|
340 |
}
|
341 |
})();
|
src/routes/settings/(nav)/assistants/[assistantId]/+page.server.ts
CHANGED
@@ -7,6 +7,7 @@ import { PUBLIC_ORIGIN, PUBLIC_SHARE_PREFIX } from "$env/static/public";
|
|
7 |
import { WEBHOOK_URL_REPORT_ASSISTANT } from "$env/static/private";
|
8 |
import { z } from "zod";
|
9 |
import type { Assistant } from "$lib/types/Assistant";
|
|
|
10 |
async function assistantOnlyIfAuthor(locals: App.Locals, assistantId?: string) {
|
11 |
const assistant = await collections.assistants.findOne({ _id: new ObjectId(assistantId) });
|
12 |
|
@@ -109,7 +110,7 @@ export const actions: Actions = {
|
|
109 |
});
|
110 |
|
111 |
if (!res.ok) {
|
112 |
-
|
113 |
}
|
114 |
}
|
115 |
|
|
|
7 |
import { WEBHOOK_URL_REPORT_ASSISTANT } from "$env/static/private";
|
8 |
import { z } from "zod";
|
9 |
import type { Assistant } from "$lib/types/Assistant";
|
10 |
+
import { logger } from "$lib/server/logger";
|
11 |
async function assistantOnlyIfAuthor(locals: App.Locals, assistantId?: string) {
|
12 |
const assistant = await collections.assistants.findOne({ _id: new ObjectId(assistantId) });
|
13 |
|
|
|
110 |
});
|
111 |
|
112 |
if (!res.ok) {
|
113 |
+
logger.error(`Webhook assistant report failed. ${res.statusText} ${res.text}`);
|
114 |
}
|
115 |
}
|
116 |
|