simeonradivoev/gameflow-deck
simeonradivoev/gameflow-deck
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: Implemented link game importing

Browse source 
feat: Implemented download page for downloading roms from various sources using plugins. Added support for internet archive external plugin.
feat: Added tasks page to track running tasks/downloads
feat: Added tanstack caching
feat: Added quick play action Fixes #6
feat: Added quick emulator launch action
fix: Made task queue only support 1 task per group and task ID should now be unique
This commit is contained in:
Simeon Radivoev 2026-05-15 13:50:55 +03:00
parent 9a3e605625
commit 9141fb35d4
Signed by: simeonradivoev
GPG key ID: 7611A451D2A5D37A
70 changed files with 1922 additions and 560 deletions
Download patch file Download diff file Expand all files Collapse all files

1
src/bun/api/controls/windows.ts
View file

@ -72,7 +72,6 @@ export class GamepadWindows implements IGamepadBackend
private index: number;
private buffer = new ArrayBuffer(16);
private view = new DataView(this.buffer);
private prevButtons = 0;
private currButtons = 0;
constructor(index = 0) { this.index = index; }

66
src/bun/api/games/games.ts
View file

@ -5,7 +5,7 @@ import z from "zod";
import * as schema from "@schema/app";
import fs from "node:fs/promises";
import { SERVER_URL } from "@shared/constants";
import { GameListFilterSchema } from '@simeonradivoev/gameflow-sdk/shared';
import { CommandEntry, DownloadLookupEntry, DownloadsLookupFilterValues, GameListFilterSchema } from '@simeonradivoev/gameflow-sdk/shared';
import { InstallJob } from "../jobs/install-job";
import path from "node:path";
import { convertLocalToFrontend, getLocalGameMatch, getSourceGameDetailed } from "./services/utils";
@ -512,7 +512,25 @@ export default new Elysia()
await plugins.hooks.games.gameLookup.promise(matches, { source, id });
return Array.from(matches.values()).flatMap(m => m);
})
.post('/game/:source/:id/play', async ({ params: { id, source }, body, set }) =>
.get('/game/:source/:id/commands', async ({ params: { id, source }, set }) =>
{
const validCommands = await getValidLaunchCommandsForGame(source, id);
if (validCommands instanceof Error)
{
return errorToResponse(validCommands, set);
}
return validCommands as {
commands: CommandEntry[];
gameId: FrontEndId;
source?: string;
sourceId?: string;
} | undefined;
}, {
response: z.object({
commands: z.custom<CommandEntry>().array()
})
})
.post('/game/:source/:id/play', async ({ params: { id, source }, body: { command_id }, set }) =>
{
const validCommands = await getValidLaunchCommandsForGame(source, id);
if (validCommands)
@ -525,7 +543,7 @@ export default new Elysia()
{
try
{
const validCommand = body.command_id ? validCommands.commands.find(c => c.id === body.command_id) : validCommands.commands[0];
const validCommand = command_id ? validCommands.commands.find(c => c.id === command_id) : validCommands.commands[0];
if (validCommand)
{
// launch command waits for the game to exit, we don't want that.
@ -676,7 +694,10 @@ export default new Elysia()
.post('/add/custom', async ({ body: { source, id, platformId, gamePath } }) =>
{
if (taskQueue.hasActiveOfType(ImportJob)) return status("Conflict", "Import Job Already Running");
const data = await taskQueue.enqueue(ImportJob.id, new ImportJob(source, id, gamePath, platformId), true);
const data = await taskQueue.enqueue(ImportJob.query({ source, id }), new ImportJob(source, id, gamePath, platformId), {
throwOnCancel: true
});
return { source: 'local', id: data.localId };
}, {
body: z.object({
@ -685,4 +706,41 @@ export default new Elysia()
gamePath: z.string(),
platformId: z.number()
})
}).get('/downloads/lookup', async ({ query: { search, page, rows, orderBy, sortDirection, source } }) =>
{
const matches = new Map<string, { count: number, items: DownloadLookupEntry[]; }>();
await plugins.hooks.games.downloadsLookup.promise(matches, { search, page, rows, orderBy, sortDirection, source });
const allValues = Array.from(matches.values());
return { hadMatchers: matches.size > 0, matches: allValues.flatMap(m => m.items), totalCount: allValues.reduce((p, c) => p + c.count, 0) };
}, {
query: z.object({
search: z.string().optional(),
page: z.coerce.number().optional(),
rows: z.coerce.number().optional(),
orderBy: z.string().optional(),
sortDirection: z.literal(["desc", "asc"]).optional(),
source: z.string().optional()
})
}).get('/download/lookup/:source/:id', async ({ params: { source, id } }) =>
{
const match = await plugins.hooks.games.downloadLookup.promise({ source, id });
if (!match) return status("Not Found");
return match;
}).get('/download/file/info', async ({ query: { file_url } }) =>
{
const response = await fetch(file_url, { method: "HEAD" });
if (!response.ok) return status('Internal Server Error', response.statusText);
return { size: Number(response.headers.get('content-length')), content_type: response.headers.get('content-type') };
}, {
query: z.object({ file_url: z.url() })
}).get('/download/lookup/filters', async () =>
{
const filters: DownloadsLookupFilterValues = {
source: [],
orderBy: []
};
await plugins.hooks.games.downloadsLookupFilters.promise({ filters });
return filters;
});

38
src/bun/api/games/services/utils.ts
View file

@ -8,7 +8,7 @@ import { RPC_URL } from "@shared/constants";
import { hashFile } from "@/bun/utils";
import { host } from "@/bun/utils/host";
import * as emulatorSchema from "@schema/emulators";
import { DownloadFileEntry, FrontEndGameType, FrontEndGameTypeDetailed, GameLookup, LocalDownloadFileEntry, LocalGameMetadata } from "@simeonradivoev/gameflow-sdk/shared";
import { DownloadFileEntry, FrontEndGameType, FrontEndGameTypeDetailed, GameLookup, LocalDownloadFileEntry, LocalGameMetadata, ProgressStats } from "@simeonradivoev/gameflow-sdk/shared";
export async function calculateSize (installPath: string | null)
{
@ -467,4 +467,40 @@ export async function createLocalGame (info: {
});
return id;
}
export async function downloadGame (ctx: {
downloads: DownloadFileEntry[],
auth?: string,
id: string,
abortSignal?: AbortSignal,
setProgress?: (progress: number, state: "download" | "extract", info: Partial<Omit<ProgressStats, 'progress'>>) => void,
extract_path?: string;
path_fs?: string;
}): Promise<string[] | undefined>
{
const downloadedFiles = await plugins.hooks.downloadFiles.promise({
id: ctx.id,
auth: ctx.auth,
files: ctx.downloads,
downloadPath: config.get('downloadPath'),
abortSignal: ctx.abortSignal,
updateProgress: (stats) => ctx.setProgress?.(stats.progress, 'download', stats)
});
if (!downloadedFiles)
{
return;
}
const finalFiles = await plugins.hooks.postDownloadFiles.promise({
files: downloadedFiles.files,
source: downloadedFiles.source,
extract_path: ctx.extract_path,
downloadPath: config.get('downloadPath'),
path_fs: ctx.path_fs
}) ?? downloadedFiles.files;
return finalFiles;
}

36
src/bun/api/jobs/bios-download-job.ts
View file

@ -1,35 +1,44 @@
import z from "zod";
import { IJob, JobContext } from "../../../packages/gameflow-sdk/task-queue";
import { IJob, JobContext } from "@simeonradivoev/gameflow-sdk/task-queue";
import { config, plugins } from "../app";
import { simulateProgress } from "@/bun/utils";
import { Downloader } from "@/bun/utils/downloader";
import path from 'node:path';
import { ensureDir } from "fs-extra";
import { buildStoreFrontendEmulatorSystems, getStoreEmulatorPackage } from "../store/services/gamesService";
import { DownloadJobData } from "@simeonradivoev/gameflow-sdk/shared";
export class BiosDownloadJob implements IJob<z.infer<typeof BiosDownloadJob.dataSchema>, "download">
interface BiosDownloadJobData extends DownloadJobData
{
emulator: string;
}
export class BiosDownloadJob implements IJob<BiosDownloadJobData, "download">
{
static id = "bios-download-job" as const;
static dataSchema = z.object({ emulator: z.string() });
static query = (q: { id: string; }) => `${BiosDownloadJob.id}-${q.id}`;
group: string = "bios-download";
emulator: string;
data: BiosDownloadJobData;
dryRun: boolean;
constructor(emulator: string, init?: { dryRun?: boolean; })
{
this.emulator = emulator;
this.data = {
emulator,
name: "Download Emulator Bios"
};
this.dryRun = init?.dryRun ?? false;
}
async start (context: JobContext<IJob<z.infer<typeof BiosDownloadJob.dataSchema>, "download">, z.infer<typeof BiosDownloadJob.dataSchema>, "download">)
async start (context: JobContext<IJob<BiosDownloadJobData, "download">, BiosDownloadJobData, "download">)
{
const emulator = await getStoreEmulatorPackage(this.emulator);
const emulator = await getStoreEmulatorPackage(this.data.emulator);
if (!emulator) throw new Error("Could Not Find Emulator");
this.data.name = `${emulator.name} Bios`;
this.data.preview_url = emulator.logo;
const systems = await buildStoreFrontendEmulatorSystems(emulator);
const biosFolder = path.join(config.get('downloadPath'), "bios", this.emulator);
const biosFolder = path.join(config.get('downloadPath'), "bios", this.data.emulator);
await ensureDir(biosFolder);
const files = await plugins.hooks.emulators.fetchBiosDownload.promise({ emulator: this.emulator, systems, biosFolder });
const files = await plugins.hooks.emulators.fetchBiosDownload.promise({ emulator: this.data.emulator, systems, biosFolder });
if (!files) throw new Error("Could not find source to download from");
@ -45,9 +54,12 @@ export class BiosDownloadJob implements IJob<z.infer<typeof BiosDownloadJob.data
const downloader = new Downloader('bios-download', files.files, biosFolder, {
signal: context.abortSignal,
headers,
onProgress (stats)
onProgress: (stats) =>
{
context.setProgress(stats.progress, "download");
this.data.downloaded = stats.downloaded;
this.data.speed = stats.speed;
this.data.total = stats.total;
},
});
@ -57,6 +69,6 @@ export class BiosDownloadJob implements IJob<z.infer<typeof BiosDownloadJob.data
exposeData ()
{
return { emulator: this.emulator };
return this.data;
}
}

52
src/bun/api/jobs/emulator-download-job.ts
View file

@ -1,14 +1,13 @@
import { EmulatorPackageType } from '@simeonradivoev/gameflow-sdk/shared';
import { DownloadJobData, EmulatorPackageType } from '@simeonradivoev/gameflow-sdk/shared';
import { getStoreEmulatorPackage } from "../store/services/gamesService";
import { IJob, JobContext } from "../../../packages/gameflow-sdk/task-queue";
import z from "zod";
import { IJob, JobContext } from "@simeonradivoev/gameflow-sdk/task-queue";
import { config, plugins } from "../app";
import path from 'node:path';
import Seven from 'node-7z';
import fs from "node:fs/promises";
import { Downloader } from "@/bun/utils/downloader";
import { ensureDir, move } from "fs-extra";
import { simulateProgress } from "@/bun/utils";
import { isArchive, simulateProgress } from "@/bun/utils";
import { path7za } from "7zip-bin";
import { getEmulatorDownload, getEmulatorPath } from "../store/services/emulatorsService";
import { $ } from "bun";
@ -16,31 +15,40 @@ import { EmulatorSourceEntryType } from "@simeonradivoev/gameflow-sdk/shared";
type EmulatorDownloadStates = "download" | "extract";
export class EmulatorDownloadJob implements IJob<z.infer<typeof EmulatorDownloadJob.dataSchema>, EmulatorDownloadStates>
interface EmulatorDownloadJobData extends DownloadJobData
{
emulator: string;
}
export class EmulatorDownloadJob implements IJob<EmulatorDownloadJobData, EmulatorDownloadStates>
{
static id = "download-emulator" as const;
static dataSchema = z.object({ emulator: z.string() });
emulator: string;
downloadSource: string;
emulatorPackage?: EmulatorPackageType;
dryRun: boolean;
isUpdate: boolean;
data: EmulatorDownloadJobData = {
name: "Download Emulator",
emulator: ""
};
constructor(emulator: string, downloadSource: string, init?: { dryRun?: boolean; isUpdate?: boolean; })
{
this.emulator = emulator;
this.data.emulator = emulator;
this.downloadSource = downloadSource;
this.dryRun = init?.dryRun ?? false;
this.isUpdate = init?.isUpdate ?? false;
}
async start (context: JobContext<EmulatorDownloadJob, z.infer<typeof EmulatorDownloadJob.dataSchema>, EmulatorDownloadStates>)
async start (context: JobContext<EmulatorDownloadJob, EmulatorDownloadJobData, EmulatorDownloadStates>)
{
this.emulatorPackage = await getStoreEmulatorPackage(this.emulator);
this.emulatorPackage = await getStoreEmulatorPackage(this.data.emulator);
if (!this.emulatorPackage) throw new Error("Emulator not found");
this.data.name = this.emulatorPackage.name;
this.data.preview_url = this.emulatorPackage.logo;
const { url, info } = await getEmulatorDownload(this.emulatorPackage, this.downloadSource);
const emulatorsFolder = getEmulatorPath(this.emulator);
const emulatorsFolder = getEmulatorPath(this.data.emulator);
if (this.dryRun)
{
@ -49,29 +57,33 @@ export class EmulatorDownloadJob implements IJob<z.infer<typeof EmulatorDownload
} else
{
const tmpFolder = path.join(config.get("downloadPath"), ".tmp");
const downloader = new Downloader(this.emulator,
[{ url, file_name: path.basename(url.pathname), file_path: this.emulator }],
const downloader = new Downloader(this.data.emulator,
[{ url, file_name: path.basename(url.pathname), file_path: this.data.emulator }],
tmpFolder,
{
signal: context.abortSignal,
onProgress (stats)
onProgress: (stats) =>
{
context.setProgress(stats.progress, 'download');
this.data.total = stats.total;
this.data.downloaded = stats.downloaded;
this.data.speed = stats.speed;
},
});
const destinationPaths = await downloader.start();
context.abortSignal.throwIfAborted();
if (destinationPaths)
{
const isArchive = destinationPaths[0].endsWith('.7z') || destinationPaths[0].endsWith('.zip') || destinationPaths[0].endsWith('.tar');
const archive = isArchive(destinationPaths[0]);
const isAppImage = destinationPaths[0].endsWith(".AppImage");
if (!isArchive && !isAppImage)
if (!archive && !isAppImage)
{
throw new Error("Invalid Download Type");
}
if (isArchive)
if (archive)
{
if (destinationPaths[0])
{
@ -120,10 +132,10 @@ export class EmulatorDownloadJob implements IJob<z.infer<typeof EmulatorDownload
await Bun.write(`${emulatorsFolder}.json`, JSON.stringify(info, null, 3));
const execs: EmulatorSourceEntryType[] = [];
await plugins.hooks.emulators.findEmulatorSource.promise({ emulator: this.emulator, sources: execs });
await plugins.hooks.emulators.findEmulatorSource.promise({ emulator: this.data.emulator, sources: execs });
await plugins.hooks.emulators.emulatorPostInstall.promise({
emulator: this.emulator,
emulator: this.data.emulator,
emulatorPackage: this.emulatorPackage,
path: execs.find(e => e.type === 'store')?.binPath ?? emulatorsFolder,
info,
@ -136,7 +148,7 @@ export class EmulatorDownloadJob implements IJob<z.infer<typeof EmulatorDownload
exposeData ()
{
return { emulator: this.emulator };
return this.data;
}
}

77
src/bun/api/jobs/import-job.ts
View file

@ -1,21 +1,32 @@
import { eq, or } from "drizzle-orm";
import { eq, inArray, or } from "drizzle-orm";
import { db, plugins } from "../app";
import { createLocalGame } from "../games/services/utils";
import { IJob, JobContext } from "../../../packages/gameflow-sdk/task-queue";
import { createLocalGame, downloadGame } from "../games/services/utils";
import { IJob, JobContext } from "@simeonradivoev/gameflow-sdk/task-queue";
import * as schema from "@schema/app";
import z from "zod";
import { GameLookup } from "@simeonradivoev/gameflow-sdk/shared";
import { DownloadJobData, GameLookup } from "@simeonradivoev/gameflow-sdk/shared";
import { isUrl } from "@/shared/utils";
import { basename } from "node:path";
import path from 'node:path';
import { isArchive } from "@/bun/utils";
export class ImportJob implements IJob<z.infer<typeof ImportJob.dataSchema>, string>
interface ImportJobData extends DownloadJobData
{
localId: number | null;
}
export class ImportJob implements IJob<ImportJobData, string>
{
static id = "import-job" as const;
static dataSchema = z.object({ localId: z.number().nullable() });
static query = (q: { source: string; id: string; }) => `${ImportJob.id}-${q.source}-${q.id}`;
data: ImportJobData = {
localId: null,
name: "Import Game"
};
group?: 'import-job';
gamePath: string;
source: string;
id: string;
platformId: number;
localId: number | null = null;
constructor(source: string, id: string, gamePath: string, platformId: number)
{
@ -25,18 +36,20 @@ export class ImportJob implements IJob<z.infer<typeof ImportJob.dataSchema>, str
this.platformId = platformId;
}
exposeData (): z.infer<typeof ImportJob.dataSchema>
exposeData ()
{
return { localId: this.localId };
return this.data;
}
async start (context: JobContext<IJob<z.infer<typeof ImportJob.dataSchema>, string>, z.infer<typeof ImportJob.dataSchema>, string>): Promise<any>
async start (context: JobContext<IJob<ImportJobData, string>, ImportJobData, string>): Promise<any>
{
const matchesMap = new Map<string, GameLookup[]>();
await plugins.hooks.games.gameLookup.promise(matchesMap, { source: this.source, id: this.id });
const matches = matchesMap.values().next().value;
if (!matches || matches.length <= 0) throw Error("Could not Find Game");
const match = matches[0];
this.data.name = match.name;
this.data.preview_url = match.coverUrl;
let cover: Buffer<ArrayBufferLike> | undefined = undefined;
let coverType: string | undefined = undefined;
@ -50,24 +63,56 @@ export class ImportJob implements IJob<z.infer<typeof ImportJob.dataSchema>, str
}
}
const platformMatch = match.platforms.find(p => p.id === this.platformId);
const finalFiles: string[] = [];
if (isUrl(this.gamePath))
{
const archive = isArchive(this.gamePath);
const downloadedFiles = await downloadGame({
downloads: [{
file_path: this.id,
file_name: basename(this.gamePath),
url: new URL(this.gamePath)
}],
extract_path: archive ? '.tmp' : undefined,
path_fs: path.join('roms', platformMatch?.slug ?? this.source, this.id),
abortSignal: context.abortSignal,
id: `game-${this.source}-${this.id}`,
setProgress: (progress, state, info) =>
{
context.setProgress(progress, state);
this.data.speed = info.speed;
this.data.total = info.total;
this.data.downloaded = info.downloaded;
},
});
if (downloadedFiles)
finalFiles.push(...downloadedFiles);
} else
{
finalFiles.push(this.gamePath);
}
const localSearchFilters: any[] = [];
if (match.igdb_id) localSearchFilters.push(eq(schema.games.igdb_id, match.igdb_id));
if (match.slug) localSearchFilters.push(eq(schema.games.slug, match.slug));
localSearchFilters.push(eq(schema.games.name, match.name));
localSearchFilters.push(eq(schema.games.path_fs, this.gamePath));
localSearchFilters.push(inArray(schema.games.path_fs, finalFiles));
const existingLocalGame = await db.query.games.findFirst({ where: or(...localSearchFilters) });
context.abortSignal.throwIfAborted();
if (existingLocalGame) throw new Error("Game Already Exists");
const platformMatch = match.platforms.find(p => p.id === this.platformId);
this.localId = await createLocalGame({
this.data.localId = await createLocalGame({
name: match.name,
system_slug: platformMatch?.slug,
source: undefined,
source_id: undefined,
slug: match.slug,
path_fs: this.gamePath,
path_fs: finalFiles[0],
summary: match.summary,
igdb_id: match.igdb_id,
ra_id: undefined,

158
src/bun/api/jobs/install-job.ts
View file

@ -1,17 +1,12 @@
import { IJob, JobContext } from "../../../packages/gameflow-sdk/task-queue";
import { IJob, JobContext } from "@simeonradivoev/gameflow-sdk/task-queue";
import fs from 'node:fs/promises';
import path from 'node:path';
import { config, events, plugins } from "../app";
import { simulateProgress } from "@/bun/utils";
import { Downloader } from "@/bun/utils/downloader";
import Seven from 'node-7z';
import z from "zod";
import { checkFiles, createLocalGame } from "../games/services/utils";
import { ensureDir, move } from "fs-extra";
import { path7za } from "7zip-bin";
import StreamZip from 'node-stream-zip';
import { which } from "bun";
import { DownloadInfo } from "@simeonradivoev/gameflow-sdk/shared";
import { checkFiles, createLocalGame, downloadGame } from "../games/services/utils";
import { ensureDir } from "fs-extra";
import { DownloadInfo, DownloadJobData } from "@simeonradivoev/gameflow-sdk/shared";
interface JobConfig
{
@ -22,7 +17,7 @@ interface JobConfig
export type InstallJobStates = 'download' | 'extract';
export class InstallJob implements IJob<never, InstallJobStates>
export class InstallJob implements IJob<DownloadJobData, InstallJobStates>
{
static id = "install-job" as const;
static query = (q: { source: string; id: string; }) => `${InstallJob.id}-${q.source}-${q.id}`;
@ -34,6 +29,9 @@ export class InstallJob implements IJob<never, InstallJobStates>
public localGameId?: number;
public group = InstallJob.id;
public localPath?: string;
data: DownloadJobData = {
name: "Install Game"
};
constructor(id: string, source: string, config?: JobConfig)
{
@ -42,7 +40,7 @@ export class InstallJob implements IJob<never, InstallJobStates>
this.source = source;
}
public async start (cx: JobContext<InstallJob, never, InstallJobStates>)
public async start (cx: JobContext<InstallJob, DownloadJobData, InstallJobStates>)
{
cx.setProgress(0, 'download');
await fs.mkdir(config.get('downloadPath'), { recursive: true });
@ -58,131 +56,31 @@ export class InstallJob implements IJob<never, InstallJobStates>
if (!info) throw new Error(`Could not find downloader for source ${this.source}`);
const files = await checkFiles(info.files, !!info.extract_path);
this.data.name = info.name;
this.data.preview_url = info.coverUrl;
const files = await checkFiles(info.files, !!info.extract_path);
if (this.config?.dryDownload !== true && files.some(f => !f.exists || !f.matches))
{
const headers: Record<string, string> = {};
if (info.auth)
headers['Authorization'] = info.auth;
const downloader = new Downloader(`game-${this.source}-${this.gameId}`,
files.filter(f => !f.exists || !f.matches),
config.get('downloadPath'),
const downloadedFiles = await downloadGame({
downloads: files.filter(f => !f.exists || !f.matches),
extract_path: info.extract_path,
path_fs: info.path_fs,
abortSignal: cx.abortSignal,
auth: info.auth,
id: `game-${this.source}-${this.gameId}`,
setProgress: (process, state, info) =>
{
signal: cx.abortSignal,
headers,
onProgress (stats)
{
cx.setProgress(stats.progress, 'download');
},
});
cx.setProgress(process, state);
this.data.downloaded = info.downloaded;
this.data.speed = info.speed;
this.data.total = info.total;
},
});
const downloadedFiles = await downloader.start();
if (!downloadedFiles)
{
return;
}
if (info.extract_path && downloadedFiles)
{
let progress = 0;
const progressDelta = 1 / downloadedFiles.length;
const extractPath = path.join(config.get('downloadPath'), info.path_fs ?? '', info.extract_path);
for (const filePath of downloadedFiles)
{
await new Promise(async (resolve, reject) =>
{
let sevenZipPath = process.env.ZIP7_PATH ?? path7za;
if (filePath.endsWith('.rar'))
{
let newPath: string | undefined;
if (process.platform === 'win32' && await fs.exists("C:\\Program Files\\7-Zip\\7z.exe"))
{
newPath = "C:\\Program Files\\7-Zip\\7z.exe";
} else
{
newPath = which('7z') ?? undefined;
}
if (!newPath)
{
await fs.rm(filePath);
reject(new Error("No RAR Support"));
return;
}
sevenZipPath = newPath;
}
let rejected = false;
const seven = Seven.extractFull(filePath, extractPath, { $bin: sevenZipPath, $progress: true });
seven.on('progress', p =>
{
cx.setProgress(progress + p.percent * progressDelta, "extract");
});
seven.on('error', e =>
{
reject(e);
rejected = true;
});
seven.on('end', async () =>
{
if (rejected) return;
await fs.rm(filePath);
resolve(true);
});
}).catch(async e =>
{
if (filePath.endsWith('.zip'))
{
cx.setProgress(0, "extract");
console.error(e);
console.warn("Could not extract", filePath, "with 7zip trying zip extractor");
await ensureDir(extractPath);
const zip = new StreamZip.async({ file: filePath });
let entryCount = await zip.entriesCount;
let entryCounter = entryCount;
zip.on('extract', (entry, outPath) =>
{
entryCounter--;
cx.setProgress(progress + (1 - (entryCounter / entryCount)) * 100 * progressDelta, "extract");
});
const count = await zip.extract(null, extractPath);
console.log(`Extracted ${count} entries`);
await zip.close();
await fs.rm(filePath);
} else
{
throw e;
}
});
progress += progressDelta * 100;
}
// check if 1 root folder we need to get rid of
const contents = await fs.readdir(extractPath);
if (contents.length === 1)
{
const stat = await fs.stat(path.join(extractPath, contents[0]));
if (stat.isDirectory())
{
console.log("Found 1 root folder, using that instead");
const tmpGameFolder = `${extractPath} (1)`;
await move(path.join(extractPath, contents[0]), tmpGameFolder, { overwrite: true });
await move(tmpGameFolder, extractPath, { overwrite: true });
}
}
finalFiles.push(extractPath);
} else
{
if (downloadedFiles)
finalFiles.push(...downloadedFiles);
}
}
if (this.config?.dryDownload === true && info.extract_path)
@ -193,7 +91,7 @@ export class InstallJob implements IJob<never, InstallJobStates>
const coverResponse = await fetch(info.coverUrl);
const cover = Buffer.from(await coverResponse.arrayBuffer());
if (cx.abortSignal.aborted) return;
cx.abortSignal.throwIfAborted();
this.localGameId = await createLocalGame({
cover,

95
src/bun/api/jobs/jobs.ts
View file

@ -6,19 +6,21 @@ import TwitchLoginJob from "./twitch-login-job";
import UpdateStoreJob from "./update-store";
import { EmulatorDownloadJob } from "./emulator-download-job";
import { getErrorMessage } from "@/bun/utils";
import { IJob } from "../../../packages/gameflow-sdk/task-queue";
import { BaseEvent, IJob } from "@simeonradivoev/gameflow-sdk/task-queue";
import { LaunchGameJob } from "./launch-game-job";
import { BiosDownloadJob } from "./bios-download-job";
import { InstallJob } from "./install-job";
import ReloadPluginsJob from "./reload-plugins-job";
import { FrontEndJob } from "@simeonradivoev/gameflow-sdk/shared";
function registerJob<
const Path extends string,
const Schema extends z.ZodTypeAny,
const Query extends z.ZodTypeAny,
Schema,
const States extends string,
T extends IJob<z.infer<Schema>, States>
> (_job: { id: Path; dataSchema: Schema; query?: (q: any) => string; } & (new (...args: any[]) => T))
> (_job: {
id: Path;
query?: (q: any) => string;
} & (new (...args: any[]) => IJob<Schema, States>))
{
return new Elysia().ws(_job.id, {
body: z.discriminatedUnion('type', [
@ -30,9 +32,9 @@ function registerJob<
type: z.literal(['data', 'started', 'progress']),
state: z.string().optional(),
progress: z.number(),
data: _job.dataSchema
data: z.custom<Schema>()
}),
z.object({ type: z.literal(['completed', 'ended']), data: _job.dataSchema }),
z.object({ type: z.literal(['completed', 'ended']), data: z.custom<Schema>() }),
z.object({ type: z.literal('waiting') }),
z.object({ type: z.literal('error'), error: z.string() })
]),
@ -42,7 +44,7 @@ function registerJob<
const job = taskQueue.findJob(jobId, _job);
if (job)
{
ws.send({ type: 'data', state: job.state, progress: job.progress, data: job.job.exposeData?.() });
ws.send({ type: 'data', state: job.state, progress: job.progress, data: job.job.exposeData?.() as Schema });
} else
{
ws.send({ type: 'waiting' });
@ -102,6 +104,83 @@ function registerJob<
}
export const jobs = new Elysia({ prefix: '/api/jobs' })
.ws('/list', {
response: z.discriminatedUnion('type', [
z.object({ type: z.literal("allJobs"), active: z.custom<FrontEndJob>().array(), queued: z.custom<FrontEndJob>().array() }),
z.object({ type: z.literal("started"), job: z.custom<FrontEndJob>() }),
z.object({ type: z.literal("progress"), job: z.custom<FrontEndJob>() }),
z.object({ type: z.literal("queued"), job: z.custom<FrontEndJob>() }),
z.object({ type: z.literal("aborted"), id: z.string() }),
z.object({ type: z.literal("ended"), id: z.string() }),
]),
body: z.discriminatedUnion('type', [
z.object({ type: z.literal("cancel"), id: z.string() })
]),
message (ws, message)
{
switch (message.type)
{
case "cancel":
taskQueue.cancelJob(message.id);
break;
}
},
open (ws)
{
ws.send({
type: 'allJobs',
active: taskQueue.getActiveJobs().map(j =>
{
const job: FrontEndJob = {
id: j.id,
data: j.job.exposeData?.(),
progress: j.progress,
state: j.state,
status: j.status
};
return job;
}),
queued: taskQueue.getQueuedJobs()?.map(j =>
{
const job: FrontEndJob = {
id: j.id,
data: j.job.exposeData?.(),
progress: j.progress,
state: j.state,
status: j.status
};
return job;
}) ?? []
});
(ws.data as any).dispose = [taskQueue.on('started', (e: BaseEvent) =>
{
ws.send({ type: "started", job: { id: e.id, data: e.job.job.exposeData?.(), progress: e.job.progress, state: e.job.state, status: e.job.status } });
}),
taskQueue.on('progress', (e: BaseEvent) =>
{
ws.send({ type: "progress", job: { id: e.id, data: e.job.job.exposeData?.(), progress: e.job.progress, state: e.job.state, status: e.job.status } });
}),
taskQueue.on('queued', (e: BaseEvent) =>
{
ws.send({ type: "queued", job: { id: e.id, data: e.job.job.exposeData?.(), progress: e.job.progress, state: e.job.state, status: e.job.status } });
}),
taskQueue.on('abort', (e: BaseEvent) =>
{
ws.send({ type: "aborted", id: e.id });
}),
taskQueue.on('ended', (e: BaseEvent) =>
{
ws.send({ type: "ended", id: e.id });
})];
},
close (ws, code, reason)
{
(ws.data as any).dispose.forEach((d: any) => d());
},
})
.use(registerJob(LaunchGameJob))
.use(registerJob(LoginJob))
.use(registerJob(TwitchLoginJob))

2
src/bun/api/jobs/launch-game-job.ts
View file

@ -1,5 +1,5 @@
import z from "zod";
import { IJob, JobContext } from "../../../packages/gameflow-sdk/task-queue";
import { IJob, JobContext } from "@simeonradivoev/gameflow-sdk/task-queue";
import { ActiveGameSchema, ActiveGameType } from "@simeonradivoev/gameflow-sdk";
import { config, db, events, plugins } from "../app";
import * as appSchema from "@schema/app";

2
src/bun/api/jobs/login-job.ts
View file

@ -1,5 +1,5 @@
import Elysia, { status } from "elysia";
import { IJob, JobContext } from "../../../packages/gameflow-sdk/task-queue";
import { IJob, JobContext } from "@simeonradivoev/gameflow-sdk/task-queue";
import { LOGIN_PORT, SERVER_URL } from "@/shared/constants";
import { host, localIp } from "@/bun/utils/host";
import cors from "@elysiajs/cors";

30
src/bun/api/jobs/test-download-job.ts Normal file
View file

@ -0,0 +1,30 @@
import { DownloadJobData } from "@simeonradivoev/gameflow-sdk/shared";
import { IJob, JobContext } from "@simeonradivoev/gameflow-sdk/task-queue";
import { sleep } from "bun";
export class TestDownloadJob implements IJob<DownloadJobData, string>
{
data: DownloadJobData = {
speed: 1686,
downloaded: 0,
total: 6615841,
name: "Test Download Job"
};
group = "test-download";
async start (context: JobContext<IJob<DownloadJobData, string>, DownloadJobData, string>): Promise<any>
{
for (let i = 0; i < 10; i++)
{
await sleep(1000);
context.setProgress(i / 10 * 100, 'download');
if (context.abortSignal.aborted) return;
}
}
exposeData (): DownloadJobData
{
return this.data;
}
}

5
src/bun/api/plugins/builtin/sources/com.simeonradivoev.gameflow.romm/romm.ts
View file

@ -6,7 +6,7 @@ import { DetailedRomSchema, getCollectionApiCollectionsIdGet, getCollectionsApiC
import { config, events } from "@/bun/api/app";
import path from 'node:path';
import fs from 'node:fs/promises';
import { hashFile, isSteamDeckGameMode } from "@/bun/utils";
import { hashFile, isArchive, isSteamDeckGameMode } from "@/bun/utils";
import { CACHE_KEYS, getOrCached } from "@/bun/api/cache";
import secrets from "@/bun/api/secrets";
import { getAuthToken } from "@/clients/romm/core/auth.gen";
@ -254,8 +254,7 @@ export default class RommIntegration implements PluginType<SettingsType>
let path_fs = path.join(rom.fs_path, rom.fs_name);
if (files.length === 1)
{
const name = files[0].file_name.toLocaleLowerCase();
if (name.endsWith('.zip') || name.endsWith('.7z') || name.endsWith('.rar'))
if (isArchive(files[0].file_name))
{
extract_path = '.';
path_fs = path.join(rom.fs_path, rom.slug ?? rom.fs_name_no_ext);

3
src/bun/api/plugins/builtin/sources/com.simeonradivoev.gameflow.store/services.ts
View file

@ -12,6 +12,7 @@ import mustache from "mustache";
import { getEmulatorDownload, getEmulatorPath } from "@/bun/api/store/services/emulatorsService";
import fs from "node:fs/promises";
import { CommandEntry, EmulatorSourceEntryType, EmulatorSystem, FrontEndEmulator, FrontEndFilterSets, FrontEndGameType, FrontEndGameTypeDetailed, SaveFileChange, EmulatorDownloadInfoType, StoreDownloadType, StoreGameType, EmulatorPackageType, EmulatorDownloadInfoSchema, StoreGameSchema } from "@simeonradivoev/gameflow-sdk/shared";
import { isUrl } from "@/shared/utils";
export async function getStoreGames (gamesManifest: any[], filter?: { limit?: number; offset?: number; })
{
@ -39,7 +40,7 @@ export async function getStoreGame (id: string)
function convertStoreMediaToPath (c: string)
{
if (c.startsWith('http'))
if (isUrl(c))
{
return `/api/romm/image?url=${encodeURIComponent(c)}`;
} else

134
src/bun/api/plugins/builtin/sources/com.simeonradivoev.gameflow.store/store.ts
View file

@ -2,7 +2,7 @@ import { PluginLoadingContextType, PluginType } from "@simeonradivoev/gameflow-s
import desc from './package.json';
import path, { } from 'node:path';
import { buildStoreFrontendEmulatorSystems, getAllStoreEmulatorPackages, getStoreEmulatorPackage, getStoreFolder } from "@/bun/api/store/services/gamesService";
import { Glob, pathToFileURL } from "bun";
import { Glob, pathToFileURL, which } from "bun";
import { and, eq } from "drizzle-orm";
import * as emulatorSchema from '@schema/emulators';
@ -13,6 +13,12 @@ import UpdateStoreJob from "@/bun/api/jobs/update-store";
import { getEmulatorDownload, getEmulatorPath } from "@/bun/api/store/services/emulatorsService";
import { buildFilters, buildLaunchCommand, buildSaves, convertStoreEmulatorToFrontend, convertStoreToFrontend, convertStoreToFrontendDetailed, getExistingStoreEmulatorDownload, getShuffledStoreGames, getStoreGame, getValidDownloads } from "./services";
import { DownloadInfo, FrontEndEmulatorDetailed, FrontEndGameTypeWithIds } from "@simeonradivoev/gameflow-sdk/shared";
import { isUrl } from "@/shared/utils";
import { Downloader } from "@/bun/utils/downloader";
import { ensureDir, move } from "fs-extra";
import StreamZip from "node-stream-zip";
import { path7za } from "7zip-bin";
import Seven from 'node-7z';
export default class RommIntegration implements PluginType
{
@ -295,7 +301,7 @@ export default class RommIntegration implements PluginType
const info: DownloadInfo = {
id: validDownload.id,
coverUrl: game.covers?.[0] ? game.covers[0].startsWith('http') ? game.covers[0] : pathToFileURL(path.join(getStoreFolder(), game.covers[0])).href : "",
coverUrl: game.covers?.[0] ? isUrl(game.covers[0]) ? game.covers[0] : pathToFileURL(path.join(getStoreFolder(), game.covers[0])).href : "",
screenshotUrls: game.screenshots ?? [],
files: [{
url: new URL(validDownload.url),
@ -325,5 +331,129 @@ export default class RommIntegration implements PluginType
return info;
});
});
ctx.hooks.downloadFiles.tapPromise(desc.name, async ({ id, files, downloadPath, abortSignal, auth, updateProgress }) =>
{
const headers: Record<string, string> = {};
if (auth)
headers['Authorization'] = auth;
const downloader = new Downloader(id,
files,
downloadPath,
{
signal: abortSignal,
headers,
onProgress: updateProgress,
});
const downloadedFiles = await downloader.start();
if (downloadedFiles)
{
return { source: desc.name, files: downloadedFiles };
}
});
ctx.hooks.postDownloadFiles.tapPromise(desc.name, async ({ files, extract_path, source, downloadPath, path_fs }) =>
{
if (extract_path && files && source === desc.name)
{
let progress = 0;
const progressDelta = 1 / files.length;
const extractPath = path.join(downloadPath, path_fs ?? '', extract_path);
for (const filePath of files)
{
await new Promise(async (resolve, reject) =>
{
let sevenZipPath = process.env.ZIP7_PATH ?? path7za;
if (filePath.endsWith('.rar'))
{
let newPath: string | undefined;
if (process.platform === 'win32' && await fs.exists("C:\\Program Files\\7-Zip\\7z.exe"))
{
newPath = "C:\\Program Files\\7-Zip\\7z.exe";
} else
{
newPath = which('7z') ?? undefined;
}
if (!newPath)
{
await fs.rm(filePath);
reject(new Error("No RAR Support"));
return;
}
sevenZipPath = newPath;
}
let rejected = false;
const seven = Seven.extractFull(filePath, extractPath, { $bin: sevenZipPath, $progress: true });
seven.on('progress', p =>
{
ctx.setProgress?.(progress + p.percent * progressDelta, "extract", {
speed: 0,
total: 0,
downloaded: 0
});
});
seven.on('error', e =>
{
reject(e);
rejected = true;
});
seven.on('end', async () =>
{
if (rejected) return;
await fs.rm(filePath);
resolve(true);
});
}).catch(async e =>
{
if (filePath.endsWith('.zip'))
{
ctx.setProgress?.(0, "extract", {});
console.error(e);
console.warn("Could not extract", filePath, "with 7zip trying zip extractor");
await ensureDir(extractPath);
const zip = new StreamZip.async({ file: filePath });
let entryCount = await zip.entriesCount;
let entryCounter = entryCount;
zip.on('extract', (entry, outPath) =>
{
entryCounter--;
ctx.setProgress?.(progress + (1 - (entryCounter / entryCount)) * 100 * progressDelta, "extract", {});
});
const count = await zip.extract(null, extractPath);
console.log(`Extracted ${count} entries`);
await zip.close();
await fs.rm(filePath);
} else
{
throw e;
}
});
progress += progressDelta * 100;
}
// check if 1 root folder we need to get rid of
const contents = await fs.readdir(extractPath);
if (contents.length === 1)
{
const stat = await fs.stat(path.join(extractPath, contents[0]));
if (stat.isDirectory())
{
console.log("Found 1 root folder, using that instead");
const tmpGameFolder = `${extractPath} (1)`;
await move(path.join(extractPath, contents[0]), tmpGameFolder, { overwrite: true });
await move(tmpGameFolder, extractPath, { overwrite: true });
}
}
return [extractPath];
}
});
}
}

6
src/bun/api/settings/settings.ts
View file

@ -10,6 +10,8 @@ import { getRelevantEmulators } from "./services";
import type { JSONSchema7 } from "json-schema";
import ReloadPluginsJob from "../jobs/reload-plugins-job";
import { pluginZodRegistry } from "../plugins/plugin-manager";
import { TestDownloadJob } from "../jobs/test-download-job";
import { randomUUIDv7 } from "bun";
export const settings = new Elysia({ prefix: '/api/settings' })
.get('/emulators/automatic', async () =>
@ -112,6 +114,10 @@ export const settings = new Elysia({ prefix: '/api/settings' })
{
return { value: plugins.plugins[decodeURIComponent(source)].config?.get(decodeURIComponent(id)) };
})
.post('/test/download', async () =>
{
taskQueue.enqueue(randomUUIDv7(), new TestDownloadJob());
})
.put('/:source/:id', async ({ params: { source, id }, body: { value } }) =>
{
const plugin = plugins.plugins[decodeURIComponent(source)];

8
src/bun/api/system.ts
View file

@ -86,6 +86,7 @@ export const system = new Elysia({ prefix: '/api/system' })
z.object({ type: z.literal('info'), data: SystemInfoSchema }),
z.object({ type: z.literal('focus') }),
z.object({ type: z.literal('loading'), progress: z.number(), state: z.string().optional() }),
z.object({ type: z.literal('activeTask'), progress: z.number().nullable() }),
z.object({ type: z.literal('loaded') }),
]),
async open (ws)
@ -94,6 +95,8 @@ export const system = new Elysia({ prefix: '/api/system' })
if (existingLoading) ws.send({ type: 'loading', progress: existingLoading.progress, state: existingLoading.state });
else ws.send({ type: 'loaded' });
ws.send({ type: 'activeTask', progress: taskQueue.getActiveJobs()[0]?.progress });
const startInfo = async () =>
{
const battery = await si.battery();
@ -116,6 +119,8 @@ export const system = new Elysia({ prefix: '/api/system' })
dispose.push(taskQueue.on('progress', e =>
{
ws.send({ type: 'activeTask', progress: e.progress });
if (e.id === ReloadPluginsJob.id)
{
ws.send({ type: "loading", progress: e.progress, state: e.state });
@ -127,6 +132,8 @@ export const system = new Elysia({ prefix: '/api/system' })
}));
dispose.push(taskQueue.on('started', e =>
{
ws.send({ type: 'activeTask', progress: 0 });
if (e.id === ReloadPluginsJob.id)
ws.send({ type: "loading", progress: e.job.progress, state: e.job.state });
else if (e.id === SelfUpdateJob.id)
@ -134,6 +141,7 @@ export const system = new Elysia({ prefix: '/api/system' })
}));
dispose.push(taskQueue.on('ended', e =>
{
ws.send({ type: 'activeTask', progress: null });
if (e.id !== ReloadPluginsJob.id && e.id !== SelfUpdateJob.id) return;
ws.send({ type: "loaded" });
}));

7
src/bun/utils.ts
View file

@ -5,6 +5,8 @@ import { config } from './api/app';
import fs from 'node:fs/promises';
import packageDef from '~/package.json';
const archiveRegex = /.(zip|rar|7zip|7z|tar|tar.gz)$/i;
export function checkRunning (pid: number)
{
try
@ -178,4 +180,9 @@ export async function moveAllFiles (srcDir: string, destDir: string)
export function getAppVersion ()
{
return process.env.VERSION_OVERRIDE ?? packageDef.version;
}
export function isArchive (path: string)
{
return archiveRegex.test(path);
}

22
src/bun/utils/downloader.ts
View file

@ -5,12 +5,7 @@ import fs from 'node:fs/promises';
import { createWriteStream } from "node:fs";
import { config, jar } from "../api/app";
import { moveAllFiles } from "../utils";
import { DownloadFileEntry } from "@simeonradivoev/gameflow-sdk/shared";
export interface ProgressStats
{
progress: number;
}
import { DownloadFileEntry, ProgressStats } from "@simeonradivoev/gameflow-sdk/shared";
interface TmpDownloadMetadata
{
@ -32,6 +27,7 @@ export class Downloader
id: string;
tmpPath: string;
tmpPathMeta: string;
downloadSpeed: number = 0;
/**
*
@ -163,10 +159,7 @@ export class Downloader
});
const totalBytes = totalSize || Number(res.headers.get("content-length")) || 0;
if (totalSize <= 0)
bytesReceived = 0;
else
bytesReceived += start;
bytesReceived += start;
const reader = res.body!.getReader();
@ -181,10 +174,11 @@ export class Downloader
if (totalBytes > 0 && this.onProgress)
{
const percent = (bytesReceived / totalBytes) * 100;
if (Date.now() - lastUpdate > 100)
const timeDelta = Date.now() - lastUpdate;
if (timeDelta > 100)
{
this.onProgress({ progress: percent });
this.downloadSpeed = this.downloadSpeed * 0.8 + Math.round(value.length / (timeDelta / 1000)) * 0.2;
this.onProgress({ progress: percent, downloaded: bytesReceived, total: totalBytes, speed: this.downloadSpeed });
lastUpdate = Date.now();
}
}
@ -194,7 +188,7 @@ export class Downloader
if (this.signal.reason === 'cancel')
{
console.log("Canceling Download and cleaning up files");
await fs.rm(this.tmpPath, { recursive: true });
await fs.rm(this.tmpPath, { recursive: true, maxRetries: 3, retryDelay: 3 });
await fs.rm(this.tmpPathMeta);
return;
}