Backend half

This commit is contained in:
2025-07-11 19:56:28 +02:00
parent fa868e7c1d
commit 8600fa7c1d
19426 changed files with 3750448 additions and 8108 deletions
@@ -0,0 +1,4 @@
import S3Transport from "./s3-transport";
export { default as S3Transport } from "./s3-transport";
export * as IS3Transport from "./s3-transport.interface";
export default S3Transport;
@@ -0,0 +1,34 @@
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.IS3Transport = exports.S3Transport = void 0;
const s3_transport_1 = __importDefault(require("./s3-transport"));
var s3_transport_2 = require("./s3-transport");
Object.defineProperty(exports, "S3Transport", { enumerable: true, get: function () { return __importDefault(s3_transport_2).default; } });
exports.IS3Transport = __importStar(require("./s3-transport.interface"));
exports.default = s3_transport_1.default;
@@ -0,0 +1,12 @@
import TransportStream from "winston-transport";
import { S3Client } from "@aws-sdk/client-s3";
import { Options, Config, StreamInfo } from "./s3-transport.interface";
declare class S3Transport extends TransportStream {
s3Client: S3Client;
s3TransportConfig: Required<Config>;
streamInfos: Map<string, StreamInfo>;
constructor(options: Options);
log(log: any, next: () => void): Promise<void>;
close(): Promise<void>;
}
export default S3Transport;
@@ -0,0 +1,89 @@
/// <reference types="node" />
import Transport from "winston-transport";
import { CompleteMultipartUploadCommandOutput, S3ClientConfig } from "@aws-sdk/client-s3";
import { PassThrough } from "stream";
/**
* Config
*/
export interface Config<T = any> {
/**
* bucket
*/
bucket: string;
/**
* generateGruop
*/
generateGruop?: (log: T) => string;
/**
* generateBucketPath
*/
generateBucketPath?: (group: string, log: T) => string;
/**
* maxBufferSize
*/
maxBufferSize?: number;
/**
* maxBufferCount
*/
maxBufferCount?: number;
/**
* maxFileSize
*/
maxFileSize?: number;
/**
* maxFileAge
*/
maxFileAge?: number;
/**
* gzip
*/
gzip?: boolean;
}
/**
* Options
*/
export interface Options extends Transport.TransportStreamOptions {
/**
* s3ClientConfig
* @see https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/interfaces/s3clientconfig.html
*/
s3ClientConfig: S3ClientConfig;
/**
* S3TransportConfig
*/
s3TransportConfig: Config;
}
/**
* StreamInfoName
*/
export declare enum StreamInfoName {
/**
* TotalWrittenBytes
*/
TotalWrittenBytes = 0,
/**
* Stream
*/
Stream = 1,
/**
* S3Upload
*/
S3Upload = 2
}
/**
* StreamInfo
*/
export type StreamInfo = [
/**
* totalWrittenBytes
*/
totalWrittenBytes: number,
/**
* stream
*/
stream: PassThrough,
/**
* s3Upload
*/
s3Upload: Promise<CompleteMultipartUploadCommandOutput>
];
@@ -0,0 +1,21 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.StreamInfoName = void 0;
/**
* StreamInfoName
*/
var StreamInfoName;
(function (StreamInfoName) {
/**
* TotalWrittenBytes
*/
StreamInfoName[StreamInfoName["TotalWrittenBytes"] = 0] = "TotalWrittenBytes";
/**
* Stream
*/
StreamInfoName[StreamInfoName["Stream"] = 1] = "Stream";
/**
* S3Upload
*/
StreamInfoName[StreamInfoName["S3Upload"] = 2] = "S3Upload";
})(StreamInfoName = exports.StreamInfoName || (exports.StreamInfoName = {}));
@@ -0,0 +1,182 @@
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const winston_transport_1 = __importDefault(require("winston-transport"));
const client_s3_1 = require("@aws-sdk/client-s3");
const lib_storage_1 = require("@aws-sdk/lib-storage");
const node_gzip_1 = require("node-gzip");
const stream_1 = require("stream");
const s3_transport_interface_1 = require("./s3-transport.interface");
class S3Transport extends winston_transport_1.default {
constructor(options) {
super(options);
this.streamInfos = new Map();
/**
* options
*/
const { s3ClientConfig, s3TransportConfig } = options;
/**
* default config values
*/
this.s3TransportConfig = Object.assign({
/**
* generateGruop
*/
generateGruop: () => "default",
/**
* generateBucketPath
*/
generateBucketPath: () => "",
/**
* maxBufferCount
*/
maxBufferCount: 50,
/**
* maxBufferSize
*/
maxBufferSize: 1024,
/**
* maxFileSize
*/
maxFileSize: 1024 * 2,
/**
* maxFileAge
*/
maxFileAge: 1000 * 60 * 5,
/**
* gzip
*/
gzip: false }, s3TransportConfig);
this.s3Client = new client_s3_1.S3Client(s3ClientConfig);
process
.on("SIGINT", () => __awaiter(this, void 0, void 0, function* () {
yield this.close();
process.exit(0);
}))
.on("beforeExit", () => __awaiter(this, void 0, void 0, function* () {
yield this.close();
}));
}
log(log, next) {
return __awaiter(this, void 0, void 0, function* () {
const { bucket, generateGruop, generateBucketPath, maxBufferSize, maxBufferCount, maxFileSize, maxFileAge, gzip, } = this.s3TransportConfig;
/**
* Generate the log group for the path.
*/
const group = generateGruop(log);
const data = `${JSON.stringify(log)}\n`;
const dataBuffer = gzip ? yield (0, node_gzip_1.gzip)(data) : Buffer.from(data);
/**
* Get the streamInfo object for the group.
*/
let groupStreamInfo = this.streamInfos.get(group);
/**
* If the buffer size exceeds the maximum buffer size, the buffer is flushed.
*/
if (groupStreamInfo &&
groupStreamInfo[s3_transport_interface_1.StreamInfoName.TotalWrittenBytes] +
dataBuffer.byteLength >=
maxFileSize) {
yield new Promise((resolve) => {
groupStreamInfo === null || groupStreamInfo === void 0 ? void 0 : groupStreamInfo[s3_transport_interface_1.StreamInfoName.Stream].end(() => {
resolve();
});
});
}
/**
* Get the streamInfo object for the group.
*/
groupStreamInfo = this.streamInfos.get(group);
if (groupStreamInfo === undefined) {
/**
* If the number of buffer size exceeds the maximum number of buffer sizes,
* the stream with the most written data is removed and a new stream is created.
*/
if (this.streamInfos.size >= maxBufferCount) {
const sortedStreamInfos = Array.from(this.streamInfos.entries()).sort((a, b) => b[1][0] - a[1][0]);
const [, firstStreamInfo] = sortedStreamInfos[0];
const [, firstStream] = firstStreamInfo;
firstStream.end();
}
/**
* Create a new streamInfo
*/
const bucketPathStream = new stream_1.PassThrough({
highWaterMark: maxBufferSize,
});
/**
* If all data in the buffer is uploaded, the stream is closed and deleted from the streams object
*/
const uploadPromise = new lib_storage_1.Upload({
client: this.s3Client,
params: {
Bucket: bucket,
Key: generateBucketPath(group, log),
Body: bucketPathStream,
},
}).done();
/**
* If the maximum file age is set,
* the stream is automatically closed after the set time.
*/
let autoFlushProcId;
if (maxFileAge > 0) {
autoFlushProcId = setTimeout(() => {
bucketPathStream.end();
}, maxFileAge);
}
uploadPromise
.then(() => {
this.streamInfos.delete(group);
clearTimeout(autoFlushProcId);
})
.catch((error) => {
// eslint-disable-next-line no-console
console.error(error);
this.streamInfos.delete(group);
clearTimeout(autoFlushProcId);
});
bucketPathStream.once("error", () => {
/**
* If an error occurs, delete the stream.
*/
this.streamInfos.delete(group);
clearTimeout(autoFlushProcId);
});
groupStreamInfo = [0, bucketPathStream, uploadPromise];
this.streamInfos.set(group, groupStreamInfo);
}
/**
* Write log data to the stream.
*/
groupStreamInfo[s3_transport_interface_1.StreamInfoName.Stream].write(dataBuffer);
groupStreamInfo[s3_transport_interface_1.StreamInfoName.TotalWrittenBytes] += dataBuffer.length;
next === null || next === void 0 ? void 0 : next();
});
}
close() {
return __awaiter(this, void 0, void 0, function* () {
/**
* Close streams.
*/
const pormiseList = [...this.streamInfos.values()].map((groupStreamInfo) => {
const [, stream, uploadPromise] = groupStreamInfo;
stream.end();
return uploadPromise;
});
yield Promise.all(pormiseList);
});
}
}
exports.default = S3Transport;