From 7a88b35ae2e988aeb8811b14344d75a583b31b05 Mon Sep 17 00:00:00 2001 From: Vitaly Turovsky Date: Thu, 28 Mar 2024 01:04:12 +0300 Subject: [PATCH 1/2] keep testing and half tested --- buildWorkers.mjs | 62 ++++++++++++++++++++++ scripts/esbuildPlugins.mjs | 95 ++++++++++++++++++---------------- src/index.ts | 39 +++++++++++++- src/react/IndicatorEffects.tsx | 1 - src/workerWorkaround.ts | 3 ++ src/worldSaveWorker.ts | 72 ++++++++++++++++++++++++++ 6 files changed, 225 insertions(+), 47 deletions(-) create mode 100644 buildWorkers.mjs create mode 100644 src/workerWorkaround.ts create mode 100644 src/worldSaveWorker.ts diff --git a/buildWorkers.mjs b/buildWorkers.mjs new file mode 100644 index 00000000..1f1782a5 --- /dev/null +++ b/buildWorkers.mjs @@ -0,0 +1,62 @@ +// main worker file intended for computing world geometry is built using prismarine-viewer/buildWorker.mjs +import { build, context } from 'esbuild' +import fs from 'fs' +import { sharedPlugins } from './scripts/esbuildPlugins.mjs' + +const watch = process.argv.includes('-w') + +const result = await (watch ? context : build)({ + bundle: true, + platform: 'browser', + entryPoints: [/* 'prismarine-viewer/examples/webglRendererWorker.ts', */'src/worldSaveWorker.ts'], + outdir: 'prismarine-viewer/public/', + sourcemap: watch ? 'inline' : 'external', + minify: !watch, + treeShaking: true, + logLevel: 'info', + alias: { + 'three': './node_modules/three/src/Three.js', + events: 'events', // make explicit + buffer: 'buffer', + 'fs': 'browserfs/dist/shims/fs.js', + http: 'http-browserify', + perf_hooks: './src/perf_hooks_replacement.js', + crypto: './src/crypto.js', + stream: 'stream-browserify', + net: 'net-browserify', + assert: 'assert', + dns: './src/dns.js' + }, + inject: [ + './src/shims.js' + ], + plugins: [ + { + name: 'writeOutput', + setup (build) { + build.onEnd(({ outputFiles }) => { + for (const file of outputFiles) { + for (const dir of ['prismarine-viewer/public', 'dist']) { + const baseName = file.path.split('/').pop() + fs.writeFileSync(`${dir}/${baseName}`, file.contents) + } + } + }) + } + }, + ...sharedPlugins + ], + loader: { + '.vert': 'text', + '.frag': 'text' + }, + mainFields: [ + 'browser', 'module', 'main' + ], + keepNames: true, + write: false, +}) + +if (watch) { + await result.watch() +} diff --git a/scripts/esbuildPlugins.mjs b/scripts/esbuildPlugins.mjs index 95d88b8e..9632e377 100644 --- a/scripts/esbuildPlugins.mjs +++ b/scripts/esbuildPlugins.mjs @@ -13,7 +13,7 @@ const prod = process.argv.includes('--prod') let connectedClients = [] /** @type {import('esbuild').Plugin[]} */ -const plugins = [ +const sharedPlugins = [ { name: 'strict-aliases', setup (build) { @@ -58,6 +58,53 @@ const plugins = [ }) } }, + { + name: 'fix-dynamic-require', + setup (build) { + build.onResolve({ + filter: /1\.14\/chunk/, + }, async ({ resolveDir, path }) => { + if (!resolveDir.includes('prismarine-provider-anvil')) return + return { + namespace: 'fix-dynamic-require', + path, + pluginData: { + resolvedPath: `${join(resolveDir, path)}.js`, + resolveDir + }, + } + }) + build.onLoad({ + filter: /.+/, + namespace: 'fix-dynamic-require', + }, async ({ pluginData: { resolvedPath, resolveDir } }) => { + const resolvedFile = await fs.promises.readFile(resolvedPath, 'utf8') + return { + contents: resolvedFile.replace("require(`prismarine-chunk/src/pc/common/BitArray${noSpan ? 'NoSpan' : ''}`)", "noSpan ? require(`prismarine-chunk/src/pc/common/BitArray`) : require(`prismarine-chunk/src/pc/common/BitArrayNoSpan`)"), + resolveDir, + loader: 'js', + } + }) + } + }, + polyfillNode({ + polyfills: { + fs: false, + dns: false, + crypto: false, + events: false, + http: false, + stream: false, + buffer: false, + perf_hooks: false, + net: false, + assert: false, + }, + }) +] + +/** @type {import('esbuild').Plugin[]} */ +const plugins = [ { name: 'data-assets', setup (build) { @@ -229,35 +276,6 @@ const plugins = [ }) } }, - { - name: 'fix-dynamic-require', - setup (build) { - build.onResolve({ - filter: /1\.14\/chunk/, - }, async ({ resolveDir, path }) => { - if (!resolveDir.includes('prismarine-provider-anvil')) return - return { - namespace: 'fix-dynamic-require', - path, - pluginData: { - resolvedPath: `${join(resolveDir, path)}.js`, - resolveDir - }, - } - }) - build.onLoad({ - filter: /.+/, - namespace: 'fix-dynamic-require', - }, async ({ pluginData: { resolvedPath, resolveDir } }) => { - const resolvedFile = await fs.promises.readFile(resolvedPath, 'utf8') - return { - contents: resolvedFile.replace("require(`prismarine-chunk/src/pc/common/BitArray${noSpan ? 'NoSpan' : ''}`)", "noSpan ? require(`prismarine-chunk/src/pc/common/BitArray`) : require(`prismarine-chunk/src/pc/common/BitArrayNoSpan`)"), - resolveDir, - loader: 'js', - } - }) - } - }, { name: 'react-displayname', setup (build) { @@ -283,20 +301,7 @@ const plugins = [ }) } }, - polyfillNode({ - polyfills: { - fs: false, - dns: false, - crypto: false, - events: false, - http: false, - stream: false, - buffer: false, - perf_hooks: false, - net: false, - assert: false, - }, - }) + ...sharedPlugins ] -export { plugins, connectedClients as clients } +export { plugins, connectedClients as clients, sharedPlugins } diff --git a/src/index.ts b/src/index.ts index 162f19a2..e9b04581 100644 --- a/src/index.ts +++ b/src/index.ts @@ -96,6 +96,7 @@ import { handleMovementStickDelta, joystickPointer } from './react/TouchAreasCon import { possiblyHandleStateVariable } from './googledrive' import flyingSquidEvents from './flyingSquidEvents' import { hideNotification, notificationProxy } from './react/NotificationProvider' +import { generateSpiralMatrix } from 'flying-squid/dist/utils' window.debug = debug window.THREE = THREE @@ -919,7 +920,43 @@ downloadAndOpenFile().then((downloadAction) => { const initialLoader = document.querySelector('.initial-loader') as HTMLElement | null if (initialLoader) { initialLoader.style.opacity = '0' - window.pageLoaded = true } +window.pageLoaded = true void possiblyHandleStateVariable() + +window.testSave = () => { + const workersNum = 5 + const workers = [] as Worker[] + + for (let i = 0; i < workersNum; i++) { + const worker = new Worker('./worldSaveWorker.js') + workers.push(worker) + } + + const chunks = generateSpiralMatrix(50) + + console.time('chunks-main') + for (const [i, worker] of workers.entries()) { + worker.postMessage({ + type: 'readChunks', + chunks: chunks.slice(i * chunks.length / workersNum, (i + 1) * chunks.length / workersNum), + folder: localServer?.options.worldFolder + '/region' + }) + } + + let finishedWorkers = 0 + + for (const worker of workers) { + // eslint-disable-next-line @typescript-eslint/no-loop-func + worker.onmessage = (msg) => { + if (msg.data.type === 'done') { + finishedWorkers++ + if (finishedWorkers === workersNum) { + console.timeEnd('chunks-main') + } + } + } + } + +} diff --git a/src/react/IndicatorEffects.tsx b/src/react/IndicatorEffects.tsx index b767cc23..b9205e23 100644 --- a/src/react/IndicatorEffects.tsx +++ b/src/react/IndicatorEffects.tsx @@ -63,7 +63,6 @@ export default ({ indicators, effects }: {indicators: typeof defaultIndicatorsSt }, [effects]) useEffect(() => { - // todo use more precise timer for each effect const interval = setInterval(() => { for (const [index, effect] of effectsRef.current.entries()) { if (effect.time === 0) { diff --git a/src/workerWorkaround.ts b/src/workerWorkaround.ts new file mode 100644 index 00000000..66d6c30f --- /dev/null +++ b/src/workerWorkaround.ts @@ -0,0 +1,3 @@ +global = globalThis +globalThis.window = globalThis +window.mcData = {} diff --git a/src/worldSaveWorker.ts b/src/worldSaveWorker.ts new file mode 100644 index 00000000..ac8b4b08 --- /dev/null +++ b/src/worldSaveWorker.ts @@ -0,0 +1,72 @@ +import './workerWorkaround' +import fs from 'fs' +import './fs2' +import { Anvil } from 'prismarine-provider-anvil' +import WorldLoader from 'prismarine-world' + +import * as browserfs from 'browserfs' +import { generateSpiralMatrix } from 'flying-squid/dist/utils' +import '../dist/mc-data/1.14' +import { oneOf } from '@zardoy/utils' + +console.log('install') +browserfs.install(window) +window.fs = fs + +onmessage = (msg) => { + globalThis.readSkylight = false + if (msg.data.type === 'readChunks') { + browserfs.configure({ + fs: 'MountableFileSystem', + options: { + '/data': { fs: 'IndexedDB' }, + }, + }, async () => { + const version = '1.14.4' + const AnvilLoader = Anvil(version) + const World = WorldLoader(version) + // const folder = '/data/worlds/Greenfield v0.5.3-3/region' + const { folder } = msg.data + const world = new World(() => { + throw new Error('Not implemented') + }, new AnvilLoader(folder)) + // const chunks = generateSpiralMatrix(20) + const { chunks } = msg.data + // const spawn = { + // x: 113, + // y: 64, + + // } + console.log('starting...') + console.time('columns') + const loadedColumns = [] as any[] + const columnToTransfarable = (chunk) => { + return { + biomes: chunk.biomes, + // blockEntities: chunk.blockEntities, + // sectionMask: chunk.sectionMask, + sections: chunk.sections, + // skyLightMask: chunk.skyLightMask, + // blockLightMask: chunk.blockLightMask, + // skyLightSections: chunk.skyLightSections, + // blockLightSections: chunk.blockLightSections + } + } + + for (const chunk of chunks) { + const column = await world.getColumn(chunk[0], chunk[1]) + if (!column) throw new Error(`Column ${chunk[0]} ${chunk[1]} not found`) + postMessage({ + column: columnToTransfarable(column) + }) + } + postMessage({ + type: 'done', + }) + + console.timeEnd('columns') + }) + } +} + +// window.fs = fs From 4cf578b7749400fccd8dcbc3e69c86f3790981ab Mon Sep 17 00:00:00 2001 From: Vitaly Turovsky Date: Fri, 29 Mar 2024 14:09:10 +0300 Subject: [PATCH 2/2] test final impl --- src/flyingSquidEvents.ts | 43 +++++++++++++++++++++++++++++++++++----- src/index.ts | 36 --------------------------------- 2 files changed, 38 insertions(+), 41 deletions(-) diff --git a/src/flyingSquidEvents.ts b/src/flyingSquidEvents.ts index 6bcde9fc..0f8ea0f8 100644 --- a/src/flyingSquidEvents.ts +++ b/src/flyingSquidEvents.ts @@ -3,10 +3,43 @@ import { chatInputValueGlobal } from './react/ChatContainer' import { showNotification } from './react/NotificationProvider' export default () => { - localServer!.on('warpsLoaded', () => { - showNotification(`${localServer!.warps.length} Warps loaded`, 'Use /warp to teleport to a warp point.', false, 'label-alt', () => { - chatInputValueGlobal.value = '/warp ' - showModal({ reactType: 'chat' }) - }) + localServer!.on('warpsLoaded', () => { + showNotification(`${localServer!.warps.length} Warps loaded`, 'Use /warp to teleport to a warp point.', false, 'label-alt', () => { + chatInputValueGlobal.value = '/warp ' + showModal({ reactType: 'chat' }) }) + }) + + localServer.loadChunksOptimized = (chunks) => { + const workersNum = 5 + const workers = [] as Worker[] + + for (let i = 0; i < workersNum; i++) { + const worker = new Worker('./worldSaveWorker.js') + workers.push(worker) + } + + console.time('chunks-main') + for (const [i, worker] of workers.entries()) { + worker.postMessage({ + type: 'readChunks', + chunks: chunks.slice(i * chunks.length / workersNum, (i + 1) * chunks.length / workersNum), + folder: localServer?.options.worldFolder + '/region' + }) + } + + let finishedWorkers = 0 + + for (const worker of workers) { + // eslint-disable-next-line @typescript-eslint/no-loop-func + worker.onmessage = (msg) => { + if (msg.data.type === 'done') { + finishedWorkers++ + if (finishedWorkers === workersNum) { + console.timeEnd('chunks-main') + } + } + } + } + } } diff --git a/src/index.ts b/src/index.ts index e9b04581..3a93a1e5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -924,39 +924,3 @@ if (initialLoader) { window.pageLoaded = true void possiblyHandleStateVariable() - -window.testSave = () => { - const workersNum = 5 - const workers = [] as Worker[] - - for (let i = 0; i < workersNum; i++) { - const worker = new Worker('./worldSaveWorker.js') - workers.push(worker) - } - - const chunks = generateSpiralMatrix(50) - - console.time('chunks-main') - for (const [i, worker] of workers.entries()) { - worker.postMessage({ - type: 'readChunks', - chunks: chunks.slice(i * chunks.length / workersNum, (i + 1) * chunks.length / workersNum), - folder: localServer?.options.worldFolder + '/region' - }) - } - - let finishedWorkers = 0 - - for (const worker of workers) { - // eslint-disable-next-line @typescript-eslint/no-loop-func - worker.onmessage = (msg) => { - if (msg.data.type === 'done') { - finishedWorkers++ - if (finishedWorkers === workersNum) { - console.timeEnd('chunks-main') - } - } - } - } - -}