mirror of
https://github.com/diced/zipline.git
synced 2025-12-12 15:50:11 -08:00
fix: scripts
This commit is contained in:
@@ -86,4 +86,6 @@ export async function exportConfig({ yml, showDefaults }: { yml?: boolean; showD
|
|||||||
|
|
||||||
console.log(`${yml ? '- ' : ''}${envVar.variable}=${envValue}`);
|
console.log(`${yml ? '- ' : ''}${envVar.variable}=${envValue}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
process.exit(0);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
|
import { bytes } from '@/lib/bytes';
|
||||||
|
import { config, reloadSettings } from '@/lib/config';
|
||||||
import { guess } from '@/lib/mimes';
|
import { guess } from '@/lib/mimes';
|
||||||
import { statSync } from 'fs';
|
import { statSync } from 'fs';
|
||||||
import { readFile, readdir } from 'fs/promises';
|
import { mkdir, readdir } from 'fs/promises';
|
||||||
import { join, parse, resolve } from 'path';
|
import { join, parse, resolve } from 'path';
|
||||||
import { reloadSettings } from '@/lib/config';
|
|
||||||
import { bytes } from '@/lib/bytes';
|
|
||||||
|
|
||||||
export async function importDir(
|
export async function importDir(
|
||||||
directory: string,
|
directory: string,
|
||||||
@@ -57,16 +57,18 @@ export async function importDir(
|
|||||||
|
|
||||||
for (let i = 0; i !== files.length; ++i) {
|
for (let i = 0; i !== files.length; ++i) {
|
||||||
const info = parse(files[i]);
|
const info = parse(files[i]);
|
||||||
|
if (info.base.startsWith('.thumbnail')) continue;
|
||||||
|
|
||||||
const mime = await guess(info.ext.replace('.', ''));
|
const mime = await guess(info.ext.replace('.', ''));
|
||||||
const { size } = statSync(join(fullPath, files[i]));
|
const { size } = statSync(join(fullPath, files[i]));
|
||||||
|
|
||||||
data[i] = {
|
data.push({
|
||||||
name: info.base,
|
name: info.base,
|
||||||
type: mime,
|
type: mime,
|
||||||
size,
|
size,
|
||||||
userId,
|
userId,
|
||||||
...(folder ? { folderId: folder } : {}),
|
...(folder ? { folderId: folder } : {}),
|
||||||
};
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!skipDb) {
|
if (!skipDb) {
|
||||||
@@ -78,16 +80,25 @@ export async function importDir(
|
|||||||
|
|
||||||
const totalSize = data.reduce((acc, file) => acc + file.size, 0);
|
const totalSize = data.reduce((acc, file) => acc + file.size, 0);
|
||||||
let completed = 0;
|
let completed = 0;
|
||||||
|
let imported = 0;
|
||||||
|
|
||||||
|
if (config.datasource.type === 'local')
|
||||||
|
await mkdir(config.datasource.local!.directory, { recursive: true });
|
||||||
|
|
||||||
|
const { getDatasource } = await import('@/lib/datasource/index.js');
|
||||||
|
const datasource = getDatasource(config);
|
||||||
|
if (!datasource) return console.error('No datasource configured');
|
||||||
|
|
||||||
|
for (let i = 0; i !== data.length; ++i) {
|
||||||
|
if (!data[i]) continue;
|
||||||
|
|
||||||
const { datasource } = await import('@/lib/datasource/index.js');
|
|
||||||
for (let i = 0; i !== files.length; ++i) {
|
|
||||||
console.log(`Uploading ${data[i].name} (${bytes(data[i].size)})...`);
|
console.log(`Uploading ${data[i].name} (${bytes(data[i].size)})...`);
|
||||||
|
|
||||||
const start = process.hrtime();
|
const start = process.hrtime();
|
||||||
|
|
||||||
const buff = await readFile(join(fullPath, files[i]));
|
await datasource.put(data[i].name, join(fullPath, files[i]), {
|
||||||
await datasource.put(data[i].name, buff, {
|
|
||||||
mimetype: data[i].type ?? 'application/octet-stream',
|
mimetype: data[i].type ?? 'application/octet-stream',
|
||||||
|
noDelete: true,
|
||||||
});
|
});
|
||||||
|
|
||||||
const diff = process.hrtime(start);
|
const diff = process.hrtime(start);
|
||||||
@@ -104,7 +115,11 @@ export async function importDir(
|
|||||||
console.log(
|
console.log(
|
||||||
`Uploaded ${data[i].name} in ${timeStr} (${bytes(data[i].size)}) ${i + 1}/${files.length} ${bytes(completed)}/${bytes(totalSize)} ${uploadSpeedStr}`,
|
`Uploaded ${data[i].name} in ${timeStr} (${bytes(data[i].size)}) ${i + 1}/${files.length} ${bytes(completed)}/${bytes(totalSize)} ${uploadSpeedStr}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
++imported;
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log('Done importing files.');
|
console.log(`Done importing ${imported} files.`);
|
||||||
|
|
||||||
|
process.exit(0);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -27,4 +27,5 @@ export async function listUsers({ extra, format, id }: { extra?: string[]; forma
|
|||||||
});
|
});
|
||||||
|
|
||||||
console.log(JSON.stringify(users, null, format ? 2 : 0));
|
console.log(JSON.stringify(users, null, format ? 2 : 0));
|
||||||
|
process.exit(0);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,4 +6,5 @@ export async function readConfig({ format }: { format: boolean }) {
|
|||||||
const { config } = await import('@/lib/config/index.js');
|
const { config } = await import('@/lib/config/index.js');
|
||||||
|
|
||||||
console.log(JSON.stringify(config, null, format ? 2 : 0));
|
console.log(JSON.stringify(config, null, format ? 2 : 0));
|
||||||
|
process.exit(0);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -34,4 +34,5 @@ export async function setUser(property: string, value: string, { id }: { id: str
|
|||||||
if (property === 'password') parsed = '*********';
|
if (property === 'password') parsed = '*********';
|
||||||
|
|
||||||
console.log(`updated user(${id}) -> ${property} = ${parsed || value}`);
|
console.log(`updated user(${id}) -> ${property} = ${parsed || value}`);
|
||||||
|
process.exit(0);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,10 +1,12 @@
|
|||||||
import { Readable } from 'stream';
|
import { Readable } from 'stream';
|
||||||
|
|
||||||
|
export type PutOptions = { mimetype?: string; noDelete?: boolean };
|
||||||
|
|
||||||
export abstract class Datasource {
|
export abstract class Datasource {
|
||||||
public name: string | undefined;
|
public name: string | undefined;
|
||||||
|
|
||||||
public abstract get(file: string): null | Readable | Promise<Readable | null>;
|
public abstract get(file: string): null | Readable | Promise<Readable | null>;
|
||||||
public abstract put(file: string, data: Buffer | string, options?: { mimetype?: string }): Promise<void>;
|
public abstract put(file: string, data: Buffer | string, options?: PutOptions): Promise<void>;
|
||||||
public abstract delete(file: string | string[]): Promise<void>;
|
public abstract delete(file: string | string[]): Promise<void>;
|
||||||
public abstract size(file: string): Promise<number>;
|
public abstract size(file: string): Promise<number>;
|
||||||
public abstract totalSize(): Promise<number>;
|
public abstract totalSize(): Promise<number>;
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import { createReadStream, existsSync } from 'fs';
|
|||||||
import { access, constants, copyFile, readdir, rename, rm, stat, writeFile } from 'fs/promises';
|
import { access, constants, copyFile, readdir, rename, rm, stat, writeFile } from 'fs/promises';
|
||||||
import { join } from 'path';
|
import { join } from 'path';
|
||||||
import { Readable } from 'stream';
|
import { Readable } from 'stream';
|
||||||
import { Datasource } from './Datasource';
|
import { Datasource, PutOptions } from './Datasource';
|
||||||
|
|
||||||
async function existsAndCanRW(path: string): Promise<boolean> {
|
async function existsAndCanRW(path: string): Promise<boolean> {
|
||||||
try {
|
try {
|
||||||
@@ -29,7 +29,7 @@ export class LocalDatasource extends Datasource {
|
|||||||
return readStream;
|
return readStream;
|
||||||
}
|
}
|
||||||
|
|
||||||
public async put(file: string, data: Buffer | string): Promise<void> {
|
public async put(file: string, data: Buffer | string, { noDelete }: PutOptions): Promise<void> {
|
||||||
const path = join(this.dir, file);
|
const path = join(this.dir, file);
|
||||||
|
|
||||||
// handles if given a path to a file, it will just move it instead of doing unecessary writes
|
// handles if given a path to a file, it will just move it instead of doing unecessary writes
|
||||||
@@ -41,7 +41,8 @@ export class LocalDatasource extends Datasource {
|
|||||||
);
|
);
|
||||||
|
|
||||||
await copyFile(data, path);
|
await copyFile(data, path);
|
||||||
await rm(data);
|
|
||||||
|
if (!noDelete) await rm(data);
|
||||||
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ import { Readable } from 'stream';
|
|||||||
import { ReadableStream } from 'stream/web';
|
import { ReadableStream } from 'stream/web';
|
||||||
import Logger, { log } from '../logger';
|
import Logger, { log } from '../logger';
|
||||||
import { randomCharacters } from '../random';
|
import { randomCharacters } from '../random';
|
||||||
import { Datasource } from './Datasource';
|
import { Datasource, PutOptions } from './Datasource';
|
||||||
|
|
||||||
function isOk(code: number) {
|
function isOk(code: number) {
|
||||||
return code >= 200 && code < 300;
|
return code >= 200 && code < 300;
|
||||||
@@ -160,13 +160,7 @@ export class S3Datasource extends Datasource {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public async put(
|
public async put(file: string, data: Buffer | string, options: PutOptions = {}): Promise<void> {
|
||||||
file: string,
|
|
||||||
data: Buffer | string,
|
|
||||||
options: {
|
|
||||||
mimetype?: string;
|
|
||||||
} = {},
|
|
||||||
): Promise<void> {
|
|
||||||
let command = new PutObjectCommand({
|
let command = new PutObjectCommand({
|
||||||
Bucket: this.options.bucket,
|
Bucket: this.options.bucket,
|
||||||
Key: this.key(file),
|
Key: this.key(file),
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ declare global {
|
|||||||
var __datasource__: Datasource;
|
var __datasource__: Datasource;
|
||||||
}
|
}
|
||||||
|
|
||||||
function getDatasource(config?: Config): void {
|
function getDatasource(config?: Config): Datasource | void {
|
||||||
if (!config) return;
|
if (!config) return;
|
||||||
|
|
||||||
const logger = log('datasource');
|
const logger = log('datasource');
|
||||||
@@ -35,6 +35,8 @@ function getDatasource(config?: Config): void {
|
|||||||
logger.error(`Datasource type ${config.datasource.type} is not supported`);
|
logger.error(`Datasource type ${config.datasource.type} is not supported`);
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return datasource;
|
||||||
}
|
}
|
||||||
|
|
||||||
datasource = global.__datasource__;
|
datasource = global.__datasource__;
|
||||||
|
|||||||
Reference in New Issue
Block a user