feat: All versions now are available offline! (#174)

cypress/e2e/index.spec.ts

@@ -14,7 +14,7 @@ const compareRenderedFlatWorld = () => {
 }
 
 const testWorldLoad = () => {
-  return cy.document().then({ timeout: 25_000 }, doc => {
+  return cy.document().then({ timeout: 35_000 }, doc => {
     return new Cypress.Promise(resolve => {
       doc.addEventListener('cypress-world-ready', resolve)
     })

experiments/decode.html

@@ -0,0 +1 @@
+<script src="decode.ts" type="module"></script>

experiments/decode.ts

@@ -0,0 +1,26 @@
+// Include the pako library
+import pako from 'pako';
+import compressedJsRaw from './compressed.js?raw'
+
+function decompressFromBase64(input) {
+    // Decode the Base64 string
+    const binaryString = atob(input);
+    const len = binaryString.length;
+    const bytes = new Uint8Array(len);
+
+    // Convert the binary string to a byte array
+    for (let i = 0; i < len; i++) {
+        bytes[i] = binaryString.charCodeAt(i);
+    }
+
+    // Decompress the byte array
+    const decompressedData = pako.inflate(bytes, { to: 'string' });
+
+    return decompressedData;
+}
+
+// Use the function
+console.time('decompress');
+const decompressedData = decompressFromBase64(compressedJsRaw);
+console.timeEnd('decompress')
+console.log(decompressedData)

prismarine-viewer/esbuild.mjs

@@ -6,20 +6,19 @@ import * as esbuild from 'esbuild'
 import { polyfillNode } from 'esbuild-plugin-polyfill-node'
 import path, { dirname, join } from 'path'
 import { fileURLToPath } from 'url'
+import childProcess from 'child_process'
+import supportedVersions from '../src/supportedVersions.mjs'
 
 const dev = process.argv.includes('-w')
 
 const __dirname = path.dirname(fileURLToPath(new URL(import.meta.url)))
 
-const mcDataPath = join(__dirname, '../dist/mc-data')
+const mcDataPath = join(__dirname, '../generated/minecraft-data-optimized.json')
 if (!fs.existsSync(mcDataPath)) {
-  // shouldn't it be in the viewer instead?
-  await import('../scripts/prepareData.mjs')
+  childProcess.execSync('tsx ../scripts/makeOptimizedMcData.mjs', { stdio: 'inherit', cwd: __dirname })
 }
 
 fs.copyFileSync(join(__dirname, 'playground.html'), join(__dirname, 'public/index.html'))
-fsExtra.copySync(mcDataPath, join(__dirname, 'public/mc-data'))
-const availableVersions = fs.readdirSync(mcDataPath).map(ver => ver.replace('.js', ''))
 
 /** @type {import('esbuild').BuildOptions} */
 const buildOptions = {
@@ -37,7 +36,7 @@ const buildOptions = {
   ],
   keepNames: true,
   banner: {
-    js: `globalThis.global = globalThis;globalThis.includedVersions = ${JSON.stringify(availableVersions)};`,
+    js: `globalThis.global = globalThis;globalThis.includedVersions = ${JSON.stringify(supportedVersions)};`,
   },
   alias: {
     events: 'events',
@@ -63,13 +62,14 @@ const buildOptions = {
         }, () => {
           const defaultVersionsObj = {}
           return {
-            contents: `window.mcData ??= ${JSON.stringify(defaultVersionsObj)};module.exports = { pc: window.mcData }`,
-            loader: 'js',
+            contents: fs.readFileSync(join(__dirname, '../src/shims/minecraftData.ts'), 'utf8'),
+            loader: 'ts',
+            resolveDir: join(__dirname, '../src/shims'),
           }
         })
         build.onEnd((e) => {
           if (e.errors.length) return
-          // fs.writeFileSync(join(__dirname, 'dist/metafile.json'), JSON.stringify(e.metafile), 'utf8')
+          fs.writeFileSync(join(__dirname, './public/metafile.json'), JSON.stringify(e.metafile), 'utf8')
         })
       }
     },

prismarine-viewer/examples/playground.ts

@@ -15,6 +15,7 @@ import { loadScript } from '../viewer/lib/utils'
 import { TWEEN_DURATION } from '../viewer/lib/entities'
 import { EntityMesh } from '../viewer/lib/entity/EntityMesh'
 import { WorldDataEmitter, Viewer } from '../viewer'
+import '../../src/getCollisionShapes'
 import { toMajorVersion } from '../../src/utils'
 
 window.THREE = THREE
@@ -65,21 +66,21 @@ async function main () {
   let continuousRender = false
 
   const { version } = params
+  await window._LOAD_MC_DATA()
   // temporary solution until web worker is here, cache data for faster reloads
-  const globalMcData = window['mcData']
-  if (!globalMcData['version']) {
-    const major = toMajorVersion(version)
-    const sessionKey = `mcData-${major}`
-    if (sessionStorage[sessionKey]) {
-      Object.assign(globalMcData, JSON.parse(sessionStorage[sessionKey]))
-    } else {
-      if (sessionStorage.length > 1) sessionStorage.clear()
-      await loadScript(`./mc-data/${major}.js`)
-      try {
-        sessionStorage[sessionKey] = JSON.stringify(Object.fromEntries(Object.entries(globalMcData).filter(([ver]) => ver.startsWith(major))))
-      } catch { }
-    }
-  }
+  // const globalMcData = window['mcData']
+  // if (!globalMcData['version']) {
+  //   const major = toMajorVersion(version)
+  //   const sessionKey = `mcData-${major}`
+  //   if (sessionStorage[sessionKey]) {
+  //     Object.assign(globalMcData, JSON.parse(sessionStorage[sessionKey]))
+  //   } else {
+  //     if (sessionStorage.length > 1) sessionStorage.clear()
+  //     try {
+  //       sessionStorage[sessionKey] = JSON.stringify(Object.fromEntries(Object.entries(globalMcData).filter(([ver]) => ver.startsWith(major))))
+  //     } catch { }
+  //   }
+  // }
 
   const mcData: IndexedData = require('minecraft-data')(version)
   window['loadedData'] = mcData

prismarine-viewer/package.json

@@ -38,5 +38,8 @@
   "optionalDependencies": {
     "canvas": "^2.11.2",
     "node-canvas-webgl": "^0.3.0"
+  },
+  "devDependencies": {
+    "live-server": "^1.2.2"
   }
 }

prismarine-viewer/viewer/lib/worldrendererCommon.ts

@@ -2,7 +2,7 @@
 import { EventEmitter } from 'events'
 import { Vec3 } from 'vec3'
 import * as THREE from 'three'
-import mcDataRaw from 'minecraft-data/data.js' // handled correctly in esbuild plugin
+import mcDataRaw from 'minecraft-data/data.js' // note: using alias
 import blocksAtlases from 'mc-assets/dist/blocksAtlases.json'
 import blocksAtlasLatest from 'mc-assets/dist/blocksAtlasLatest.png'
 import blocksAtlasLegacy from 'mc-assets/dist/blocksAtlasLegacy.png'
@@ -223,8 +223,12 @@ export abstract class WorldRendererCommon<WorkerSend = any, WorkerReceive = any>
 
   sendMesherMcData () {
     const allMcData = mcDataRaw.pc[this.version] ?? mcDataRaw.pc[toMajorVersion(this.version)]
-    const mcData = Object.fromEntries(Object.entries(allMcData).filter(([key]) => dynamicMcDataFiles.includes(key)))
-    mcData.version = JSON.parse(JSON.stringify(mcData.version))
+    const mcData = {
+      version: JSON.parse(JSON.stringify(allMcData.version))
+    }
+    for (const key of dynamicMcDataFiles) {
+      mcData[key] = allMcData[key]
+    }
 
     for (const worker of this.workers) {
       worker.postMessage({ type: 'mcData', mcData, config: this.mesherConfig })

rsbuild.config.ts

@@ -101,9 +101,10 @@ export default defineConfig({
                 const prep = async () => {
                     console.time('total-prep')
                     fs.mkdirSync('./generated', { recursive: true })
-                    if (!fs.existsSync('./generated/minecraft-data-data.js')) {
-                        childProcess.execSync('tsx ./scripts/genShims.ts', { stdio: 'inherit' })
+                    if (!fs.existsSync('./generated/minecraft-data-optimized.json') || require('./generated/minecraft-data-optimized.json').versionKey !== require('minecraft-data/package.json').version) {
+                        childProcess.execSync('tsx ./scripts/makeOptimizedMcData.mjs', { stdio: 'inherit' })
                     }
+                    childProcess.execSync('tsx ./scripts/genShims.ts', { stdio: 'inherit' })
                     if (!fs.existsSync('./generated/latestBlockCollisionsShapes.json')) {
                         childProcess.execSync('tsx ./scripts/optimizeBlockCollisions.ts', { stdio: 'inherit' })
                     }
@@ -117,7 +118,6 @@ export default defineConfig({
                         configJson.defaultProxy = ':8080'
                     }
                     fs.writeFileSync('./dist/config.json', JSON.stringify(configJson), 'utf8')
-                    childProcess.execSync('node ./scripts/prepareData.mjs', { stdio: 'inherit' })
                     // childProcess.execSync('./scripts/prepareSounds.mjs', { stdio: 'inherit' })
                     // childProcess.execSync('tsx ./scripts/genMcDataTypes.ts', { stdio: 'inherit' })
                     // childProcess.execSync('tsx ./scripts/genPixelartTypes.ts', { stdio: 'inherit' })
@@ -164,7 +164,7 @@ export default defineConfig({
                     // throw new Error(`${resource.request} was requested by ${resource.contextInfo.issuer}`)
                 }
                 if (absolute.endsWith('/minecraft-data/data.js')) {
-                    resource.request = path.join(__dirname, './generated/minecraft-data-data.js')
+                    resource.request = path.join(__dirname, './src/shims/minecraftData.ts')
                 }
             }))
             addRules([

scripts/build.js

@@ -45,7 +45,6 @@ exports.getSwAdditionalEntries = () => {
     // need to be careful with this
     const filesToCachePatterns = [
         'index.html',
-        `mc-data/${defaultLocalServerOptions.versionMajor}.js`,
         'background/**',
         // todo-low copy from assets
         '*.mp3',

scripts/genShims.ts

@@ -1,24 +1,7 @@
 import fs from 'fs'
-import MinecraftData from 'minecraft-data'
-import MCProtocol from 'minecraft-protocol'
 import { appReplacableResources } from '../src/resourcesSource'
 
-const { supportedVersions, defaultVersion } = MCProtocol
-
-// gen generated/minecraft-data-data.js
-
-const data = MinecraftData(defaultVersion)
-const defaultVersionObj = {
-  [defaultVersion]: {
-    version: data.version,
-    protocol: data.protocol,
-  }
-}
-
-const mcDataContents = `window.mcData ??= ${JSON.stringify(defaultVersionObj)};module.exports = { pc: window.mcData }`
-
 fs.mkdirSync('./generated', { recursive: true })
-fs.writeFileSync('./generated/minecraft-data-data.js', mcDataContents, 'utf8')
 
 // app resources
 

scripts/makeOptimizedMcData.mjs

@@ -0,0 +1,230 @@
+//@ts-check
+import { build } from 'esbuild'
+import { existsSync } from 'node:fs'
+import Module from "node:module"
+import { dirname } from 'node:path'
+import supportedVersions from '../src/supportedVersions.mjs'
+import { gzipSizeFromFileSync } from 'gzip-size'
+import fs from 'fs'
+import  {default as _JsonOptimizer}  from '../src/optimizeJson'
+import { gzipSync } from 'zlib';
+import MinecraftData from 'minecraft-data'
+import MCProtocol from 'minecraft-protocol'
+
+/** @type {typeof _JsonOptimizer} */
+//@ts-ignore
+const JsonOptimizer = _JsonOptimizer.default
+
+// console.log(a.diff_main(JSON.stringify({ a: 1 }), JSON.stringify({ a: 1, b: 2 })))
+
+const require = Module.createRequire(import.meta.url)
+
+const dataPaths = require('minecraft-data/minecraft-data/data/dataPaths.json')
+
+function toMajor (version) {
+  const [a, b] = (version + '').split('.')
+  return `${a}.${b}`
+}
+
+const versions = {}
+const dataTypes = new Set()
+
+for (const [version, dataSet] of Object.entries(dataPaths.pc)) {
+  if (!supportedVersions.includes(version)) continue
+  for (const type of Object.keys(dataSet)) {
+    dataTypes.add(type)
+  }
+  versions[version] = dataSet
+}
+
+const versionToNumber = (ver) => {
+  const [x, y = '0', z = '0'] = ver.split('.')
+  return +`${x.padStart(2, '0')}${y.padStart(2, '0')}${z.padStart(2, '0')}`
+}
+
+// if not included here (even as {}) will not be bundled & accessible!
+const compressedOutput = false
+// const dataTypeBundling = {
+//   protocol: {
+//     // ignoreRemoved: true,
+//     // ignoreChanges: true
+//   }
+// }
+const dataTypeBundling = {
+  language: {
+    ignoreRemoved: true,
+    ignoreChanges: true
+  },
+  blocks: {
+    arrKey: 'name',
+    // ignoreRemoved: true,
+    // genChanges (source, diff) {
+    //   const diffs = {}
+    //   const newItems = {}
+    //   for (const [key, val] of Object.entries(diff)) {
+    //     const src = source[key]
+    //     if (!src) {
+    //       newItems[key] = val
+    //       continue
+    //     }
+    //     const { minStateId, defaultState, maxStateId } = val
+    //     if (defaultState === undefined || minStateId === src.minStateId || maxStateId === src.maxStateId || defaultState === src.defaultState) continue
+    //     diffs[key] = [minStateId, defaultState, maxStateId]
+    //   }
+    //   return {
+    //     stateChanges: diffs
+    //   }
+    // },
+    // ignoreChanges: true
+  },
+  items: {
+    arrKey: 'name'
+  },
+  attributes: {
+    arrKey: 'name'
+  },
+  particles: {
+    arrKey: 'name'
+  },
+  effects: {
+    arrKey: 'name'
+  },
+  enchantments: {
+    arrKey: 'name'
+  },
+  instruments: {
+    arrKey: 'name'
+  },
+  foods: {
+    arrKey: 'name'
+  },
+  entities: {
+    arrKey: 'id+type'
+  },
+  materials: {},
+  windows: {
+    arrKey: 'name'
+  },
+  version: {
+    raw: true
+  },
+  tints: {},
+  biomes: {
+    arrKey: 'name'
+  },
+  entityLoot: {
+    arrKey: 'entity'
+  },
+  blockLoot: {
+    arrKey: 'block'
+  },
+  recipes: {}, // todo we can do better
+  blockCollisionShapes: {},
+  loginPacket: {},
+  protocol: {
+    raw: true
+  },
+  sounds: {
+    arrKey: 'name'
+  }
+}
+
+const notBundling = [...dataTypes.keys()].filter(x => !Object.keys(dataTypeBundling).includes(x))
+console.log("Not bundling minecraft-data data:", notBundling)
+
+let previousData = {}
+// /** @type {Record<string, JsonOptimizer>} */
+const diffSources = {}
+const versionsArr = Object.entries(versions)
+const sizePerDataType = {}
+const rawDataVersions = {}
+// const versionsArr = Object.entries(versions).slice(-1)
+for (const [i, [version, dataSet]] of versionsArr.reverse().entries()) {
+  for (const [dataType, dataPath] of Object.entries(dataSet)) {
+    const config = dataTypeBundling[dataType]
+    if (!config) continue
+    if (dataType === 'blockCollisionShapes' && versionToNumber(version) >= versionToNumber('1.13')) {
+      // contents += `      get ${dataType} () { return window.globalGetCollisionShapes?.("${version}") },\n`
+      continue
+    }
+    const loc = `minecraft-data/data/${dataPath}/`
+    const dataPathAbsolute = require.resolve(`minecraft-data/${loc}${dataType}`)
+    // const data = fs.readFileSync(dataPathAbsolute, 'utf8')
+    const dataRaw = require(dataPathAbsolute)
+    let injectCode = ''
+    let rawData = dataRaw
+    if (config.raw) {
+      rawDataVersions[dataType] ??= {}
+      rawDataVersions[dataType][version] = rawData
+      rawData = dataRaw
+    } else {
+      if (!diffSources[dataType]) {
+        diffSources[dataType] = new JsonOptimizer(config.arrKey, config.ignoreChanges, config.ignoreRemoved)
+      }
+      try {
+        diffSources[dataType].recordDiff(version, dataRaw)
+        injectCode = `restoreDiff(sources, ${JSON.stringify(dataType)}, ${JSON.stringify(version)})`
+      } catch (err) {
+        const error = new Error(`Failed to diff ${dataType} for ${version}: ${err.message}`)
+        error.stack = err.stack
+        throw error
+      }
+    }
+    sizePerDataType[dataType] ??= 0
+    sizePerDataType[dataType] += Buffer.byteLength(JSON.stringify(injectCode || rawData), 'utf8')
+    if (config.genChanges && previousData[dataType]) {
+      const changes = config.genChanges(previousData[dataType], dataRaw)
+      // Object.assign(data, changes)
+    }
+    previousData[dataType] = dataRaw
+  }
+}
+const sources = Object.fromEntries(Object.entries(diffSources).map(x => {
+  const data = x[1].export()
+  // const data = {}
+  sizePerDataType[x[0]] += Buffer.byteLength(JSON.stringify(data), 'utf8')
+  return [x[0], data]
+}))
+Object.assign(sources, rawDataVersions)
+sources.versionKey = require('minecraft-data/package.json').version
+
+const totalSize = Object.values(sizePerDataType).reduce((acc, val) => acc + val, 0)
+console.log('total size (mb)', totalSize / 1024 / 1024)
+console.log(
+  'size per data type (mb, %)',
+  Object.fromEntries(Object.entries(sizePerDataType).map(([dataType, size]) => {
+    return [dataType, [size / 1024 / 1024, Math.round(size / totalSize * 100)]];
+  }).sort((a, b) => {
+    //@ts-ignore
+    return b[1][1] - a[1][1];
+  }))
+)
+
+function compressToBase64(input) {
+  const buffer = gzipSync(input);
+  return buffer.toString('base64');
+}
+
+const filePath = './generated/minecraft-data-optimized.json'
+fs.writeFileSync(filePath, JSON.stringify(sources), 'utf8')
+if (compressedOutput) {
+  const minizedCompressed = compressToBase64(fs.readFileSync(filePath))
+  console.log('size of compressed', Buffer.byteLength(minizedCompressed, 'utf8') / 1000 / 1000)
+  const compressedFilePath = './experiments/compressed.js'
+  fs.writeFileSync(compressedFilePath, minizedCompressed, 'utf8')
+}
+
+console.log('size', fs.lstatSync(filePath).size / 1000 / 1000, gzipSizeFromFileSync(filePath) / 1000 / 1000)
+
+// always bundled
+
+const { defaultVersion } = MCProtocol
+const data = MinecraftData(defaultVersion)
+const initialMcData = {
+  [defaultVersion]: {
+    version: data.version,
+    protocol: data.protocol,
+  }
+}
+
+fs.writeFileSync('./generated/minecraft-initial-data.json', JSON.stringify(initialMcData), 'utf8')

scripts/prepareData.mjs

@@ -1,72 +0,0 @@
-//@ts-check
-import { build } from 'esbuild'
-import { existsSync } from 'node:fs'
-import Module from "node:module"
-import { dirname } from 'node:path'
-import supportedVersions from '../src/supportedVersions.mjs'
-
-if (existsSync('dist/mc-data') && !process.argv.includes('-f')) {
-  console.log('using cached prepared data')
-  process.exit(0)
-}
-
-const require = Module.createRequire(import.meta.url)
-
-const dataPaths = require('minecraft-data/minecraft-data/data/dataPaths.json')
-
-function toMajor (version) {
-  const [a, b] = (version + '').split('.')
-  return `${a}.${b}`
-}
-
-const grouped = {}
-
-for (const [version, data] of Object.entries(dataPaths.pc)) {
-  if (!supportedVersions.includes(version)) continue
-  const major = toMajor(version)
-  grouped[major] ??= {}
-  grouped[major][version] = data
-}
-
-const versionToNumber = (ver) => {
-  const [x, y = '0', z = '0'] = ver.split('.')
-  return +`${x.padStart(2, '0')}${y.padStart(2, '0')}${z.padStart(2, '0')}`
-}
-
-console.log('preparing data')
-console.time('data prepared')
-let builds = []
-for (const [major, versions] of Object.entries(grouped)) {
-  // if (major !== '1.19') continue
-  let contents = 'Object.assign(window.mcData, {\n'
-  for (const [version, dataSet] of Object.entries(versions)) {
-    contents += `    '${version}': {\n`
-    for (const [dataType, dataPath] of Object.entries(dataSet)) {
-      if (dataType === 'blockCollisionShapes' && versionToNumber(version) >= versionToNumber('1.13')) {
-        contents += `      get ${dataType} () { return window.globalGetCollisionShapes?.("${version}") },\n`
-        continue
-      }
-      const loc = `minecraft-data/data/${dataPath}/`
-      contents += `      get ${dataType} () { return require("./${loc}${dataType}.json") },\n`
-    }
-    contents += '    },\n'
-  }
-  contents += '})'
-
-  const promise = build({
-    bundle: true,
-    outfile: `dist/mc-data/${major}.js`,
-    stdin: {
-      contents,
-
-      resolveDir: dirname(require.resolve('minecraft-data')),
-      sourcefile: `mcData${major}.js`,
-      loader: 'js',
-    },
-    metafile: true,
-  })
-  // require('fs').writeFileSync('dist/mc-data/metafile.json', JSON.stringify(promise.metafile), 'utf8')
-  builds.push(promise)
-}
-await Promise.all(builds)
-console.timeEnd('data prepared')

scripts/testOptimizedMcdata.ts

@@ -0,0 +1,99 @@
+import assert from 'assert'
+import JsonOptimizer from '../src/optimizeJson';
+import fs from 'fs'
+import minecraftData from 'minecraft-data'
+
+const json = JSON.parse(fs.readFileSync('./generated/minecraft-data-optimized.json', 'utf8'))
+
+const dataPaths = require('minecraft-data/minecraft-data/data/dataPaths.json')
+
+const validateData = (ver, type) => {
+  const target = JsonOptimizer.restoreData(json[type], ver)
+  const arrKey = json[type].arrKey
+  const originalPath = dataPaths.pc[ver][type]
+  const original = require(`minecraft-data/minecraft-data/data/${originalPath}/${type}.json`)
+  if (arrKey) {
+    const originalKeys = original.map(a => JsonOptimizer.getByArrKey(a, arrKey)) as string[]
+    for (const [i, item] of originalKeys.entries()) {
+      if (originalKeys.indexOf(item) !== i) {
+        console.warn(`${type} ${ver} Incorrect source, duplicated arrKey (${arrKey}) ${item}. Ignoring!`) // todo should span instead
+        const index = originalKeys.indexOf(item);
+        original.splice(index, 1)
+        originalKeys.splice(index, 1)
+      }
+    }
+    // if (target.length !== originalKeys.length) {
+    //   throw new Error(`wrong arr length: ${target.length} !== ${original.length}`)
+    // }
+    checkKeys(originalKeys, target.map(a => JsonOptimizer.getByArrKey(a, arrKey)))
+    for (const item of target as any[]) {
+      const keys = Object.entries(item).map(a => a[0])
+      const origItem = original.find(a => JsonOptimizer.getByArrKey(a, arrKey) === JsonOptimizer.getByArrKey(item, arrKey));
+      const keysSource = Object.entries(origItem).map(a => a[0])
+      checkKeys(keysSource, keys, true, 'prop keys', true)
+      checkObj(origItem, item)
+    }
+  } else {
+    const keysOriginal = Object.keys(original)
+    const keysTarget = Object.keys(target)
+    checkKeys(keysOriginal, keysTarget)
+    for (const key of keysTarget) {
+      checkObj(original[key], target[key])
+    }
+  }
+}
+
+const checkObj = (source, diffing) => {
+  checkKeys(Object.keys(source), Object.keys(diffing))
+  for (const [key, val] of Object.entries(source)) {
+    if (JSON.stringify(val) !== JSON.stringify(diffing[key])) {
+      throw new Error(`different value of ${key}: ${val} ${diffing[key]}`)
+    }
+  }
+}
+
+const checkKeys = (source, diffing, isUniq = true, msg = '', redunantOk = false) => {
+  if (isUniq) {
+    for (const [i, item] of diffing.entries()) {
+      if (diffing.indexOf(item) !== i) {
+        throw new Error(`Duplicate: ${item}: ${i} ${diffing.indexOf(item)} ${msg}`)
+      }
+    }
+  }
+  for (const key of source) {
+    if (!diffing.includes(key)) {
+      throw new Error(`Diffing does not include "${key}" (${msg})`)
+    }
+  }
+  if (!redunantOk) {
+    for (const key of diffing) {
+      if (!source.includes(key)) {
+        throw new Error(`Source does not include "${key}" (${msg})`)
+      }
+    }
+  }
+}
+
+// const data = minecraftData('1.20.4')
+const oldId = JsonOptimizer.restoreData(json['blocks'], '1.20').find(x => x.name === 'brown_stained_glass').id;
+const newId = JsonOptimizer.restoreData(json['blocks'], '1.20.4').find(x => x.name === 'brown_stained_glass').id;
+assert(oldId !== newId)
+// test all types + all versions
+
+for (const type of Object.keys(json)) {
+  if (!json[type].__IS_OPTIMIZED__) continue
+  if (type === 'language') continue // we have loose data for language for size reasons
+  console.log('validating', type)
+  const source = json[type]
+  let checkedVer = 0
+  for (const ver of Object.keys(source.diffs)) {
+    try {
+      validateData(ver, type)
+    } catch (err) {
+      err.message = `Failed to validate ${type} for ${ver}: ${err.message}`
+      throw err;
+    }
+    checkedVer++
+  }
+  console.log('Checked versions:', checkedVer)
+}

src/getCollisionInteractionShapes.ts

@@ -0,0 +1,17 @@
+import { getRenamedData } from 'flying-squid/dist/blockRenames'
+import outputInteractionShapesJson from './interactionShapesGenerated.json'
+import './getCollisionShapes'
+
+export default () => {
+  customEvents.on('gameLoaded', () => {
+    // todo also remap block states (e.g. redstone)!
+    const renamedBlocksInteraction = getRenamedData('blocks', Object.keys(outputInteractionShapesJson), '1.20.2', bot.version)
+    const interactionShapes = {
+      ...outputInteractionShapesJson,
+      ...Object.fromEntries(Object.entries(outputInteractionShapesJson).map(([block, shape], i) => [renamedBlocksInteraction[i], shape]))
+    }
+    interactionShapes[''] = interactionShapes['air']
+    // todo make earlier
+    window.interactionShapes = interactionShapes
+  })
+}

src/getCollisionShapes.ts

@@ -1,6 +1,5 @@
 import { getRenamedData } from 'flying-squid/dist/blockRenames'
 import collisionShapesInit from '../generated/latestBlockCollisionsShapes.json'
-import outputInteractionShapesJson from './interactionShapesGenerated.json'
 
 // defining globally to be used in loaded data, not sure of better workaround
 window.globalGetCollisionShapes = (version) => {
@@ -13,17 +12,3 @@ window.globalGetCollisionShapes = (version) => {
   }
   return collisionShapes
 }
-
-export default () => {
-  customEvents.on('gameLoaded', () => {
-    // todo also remap block states (e.g. redstone)!
-    const renamedBlocksInteraction = getRenamedData('blocks', Object.keys(outputInteractionShapesJson), '1.20.2', bot.version)
-    const interactionShapes = {
-      ...outputInteractionShapesJson,
-      ...Object.fromEntries(Object.entries(outputInteractionShapesJson).map(([block, shape], i) => [renamedBlocksInteraction[i], shape]))
-    }
-    interactionShapes[''] = interactionShapes['air']
-    // todo make earlier
-    window.interactionShapes = interactionShapes
-  })
-}

src/index.ts

@@ -5,7 +5,7 @@ import './globals'
 import './devtools'
 import './entities'
 import './globalDomListeners'
-import initCollisionShapes from './getCollisionShapes'
+import initCollisionShapes from './getCollisionInteractionShapes'
 import { onGameLoad } from './inventoryWindows'
 import { supportedVersions } from 'minecraft-protocol'
 import protocolMicrosoftAuth from 'minecraft-protocol/src/client/microsoftAuth'
@@ -380,6 +380,7 @@ async function connect (connectOptions: ConnectOptions) {
   try {
     const serverOptions = defaultsDeep({}, connectOptions.serverOverrides ?? {}, options.localServerOptions, defaultServerOptions)
     Object.assign(serverOptions, connectOptions.serverOverridesFlat ?? {})
+    window._LOAD_MC_DATA() // start loading data (if not loaded yet)
     const downloadMcData = async (version: string) => {
       if (connectOptions.authenticatedAccount && versionToNumber(version) < versionToNumber('1.19.4')) {
         // todo support it (just need to fix .export crash)
@@ -392,13 +393,13 @@ async function connect (connectOptions: ConnectOptions) {
         // ignore cache hit
         versionsByMinecraftVersion.pc[lastVersion]!['dataVersion']!++
       }
+      setLoadingScreenStatus(`Loading data for ${version}`)
       if (!document.fonts.check('1em mojangles')) {
         // todo instead re-render signs on load
         await document.fonts.load('1em mojangles').catch(() => { })
       }
-      setLoadingScreenStatus(`Downloading data for ${version}`)
+      await window._MC_DATA_RESOLVER.promise // ensure data is loaded
       await downloadSoundsIfNeeded()
-      await loadScript(`./mc-data/${toMajorVersion(version)}.js`)
       miscUiState.loadedDataVersion = version
       try {
         await resourcepackReload(version)

src/optimizeJson.ts

@@ -0,0 +1,264 @@
+import { versionToNumber } from 'prismarine-viewer/viewer/prepare/utils'
+
+type IdMap = Record<string, number>
+
+type DiffData = {
+  removed: number[],
+  changed: any[],
+  removedProps: Array<[number, number[]]>,
+  added
+}
+
+type SourceData = {
+  keys: IdMap,
+  properties: IdMap
+  source: Record<number, any>
+  diffs: Record<string, DiffData>
+  arrKey?
+  __IS_OPTIMIZED__: true
+}
+
+export default class JsonOptimizer {
+  keys = {} as IdMap
+  idToKey = {} as Record<number, string>
+  properties = {} as IdMap
+  source = {}
+  previousKeys = [] as number[]
+  previousValues = {} as Record<number, any>
+  diffs = {} as Record<string, DiffData>
+
+  constructor (public arrKey?: string, public ignoreChanges = false, public ignoreRemoved = false) { }
+
+  export () {
+    const { keys, properties, source, arrKey, diffs } = this
+    return {
+      keys,
+      properties,
+      source,
+      arrKey,
+      diffs,
+      '__IS_OPTIMIZED__': true
+    } satisfies SourceData
+  }
+
+  diffObj (diffing): DiffData {
+    const removed = [] as number[]
+    const changed = [] as any[]
+    const removedProps = [] as any[]
+    const { arrKey, ignoreChanges, ignoreRemoved } = this
+    const added = [] as number[]
+
+    if (!diffing || typeof diffing !== 'object') throw new Error('diffing data is not object')
+    if (Array.isArray(diffing) && !arrKey) throw new Error('arrKey is required for arrays')
+    const diffingObj = Array.isArray(diffing) ? Object.fromEntries(diffing.map(x => {
+      const key = JsonOptimizer.getByArrKey(x, arrKey!)
+      return [key, x]
+    })) : diffing
+
+    const possiblyNewKeys = Object.keys(diffingObj)
+    this.keys ??= {}
+    this.properties ??= {}
+    let lastRootKeyId = Object.values(this.keys).length
+    let lastItemKeyId = Object.values(this.properties).length
+    for (const key of possiblyNewKeys) {
+      this.keys[key] ??= lastRootKeyId++
+      this.idToKey[this.keys[key]] = key
+    }
+    const DEBUG = false
+
+    const addDiff = (key, newVal, prevVal) => {
+      const valueMapped = [] as any[]
+      const isItemObj = typeof newVal === 'object' && newVal
+      const keyId = this.keys[key]
+      if (isItemObj) {
+        const removedPropsLocal = [] as any[]
+        for (const [prop, val] of Object.entries(newVal)) {
+          // mc-data: why push only changed props? eg for blocks only stateId are different between all versions so we skip a lot of duplicated data like block props
+          if (!isEqualStructured(newVal[prop], prevVal[prop])) {
+            let keyMapped = this.properties[prop]
+            if (keyMapped === undefined) {
+              this.properties[prop] = lastItemKeyId++
+              keyMapped = this.properties[prop]
+            }
+            valueMapped.push(DEBUG ? prop : keyMapped, newVal[prop])
+          }
+        }
+        // also add undefined for removed props
+        for (const prop of Object.keys(prevVal)) {
+          if (prop in newVal) continue
+          let keyMapped = this.properties[prop]
+          if (keyMapped === undefined) {
+            this.properties[prop] = lastItemKeyId++
+            keyMapped = this.properties[prop]
+          }
+          removedPropsLocal.push(DEBUG ? prop : keyMapped)
+        }
+        removedProps.push([keyId, removedPropsLocal])
+      }
+      changed.push(DEBUG ? key : keyId, isItemObj ? valueMapped : newVal)
+    }
+    for (const [id, sourceVal] of Object.entries(this.source)) {
+      const key = this.idToKey[id]
+      const diffVal = diffingObj[key]
+      if (!ignoreChanges && diffVal !== undefined) {
+        this.previousValues[id] ??= this.source[id]
+        const prevVal = this.previousValues[id]
+        if (!isEqualStructured(prevVal, diffVal)) {
+          addDiff(key, diffVal, prevVal)
+        }
+        this.previousValues[id] = diffVal
+      }
+    }
+    for (const [key, val] of Object.entries(diffingObj)) {
+      const id = this.keys[key]
+      if (!this.source[id]) {
+        this.source[id] = val
+      }
+      added.push(id)
+    }
+
+    for (const previousKey of this.previousKeys) {
+      const key = this.idToKey[previousKey]
+      if (diffingObj[key] === undefined && !ignoreRemoved) {
+        removed.push(previousKey)
+      }
+    }
+
+    for (const toRemove of removed) {
+      this.previousKeys.splice(this.previousKeys.indexOf(toRemove), 1)
+    }
+
+    for (const previousKey of this.previousKeys) {
+      const index = added.indexOf(previousKey)
+      if (index === -1) continue
+      added.splice(index, 1)
+    }
+
+    this.previousKeys = [...this.previousKeys, ...added]
+
+    return {
+      removed,
+      changed,
+      added,
+      removedProps
+    }
+  }
+
+  recordDiff (key: string, diffObj: string) {
+    const diff = this.diffObj(diffObj)
+    this.diffs[key] = diff
+  }
+
+  static isOptimizedChangeDiff (changePossiblyArrDiff) {
+    if (!Array.isArray(changePossiblyArrDiff)) return false
+    if (changePossiblyArrDiff.length % 2 !== 0) return false
+    for (let i = 0; i < changePossiblyArrDiff.length; i += 2) {
+      if (typeof changePossiblyArrDiff[i] !== 'number') return false
+    }
+    return true
+  }
+
+  static restoreData ({ keys, properties, source, arrKey, diffs }: SourceData, targetKey: string) {
+    // if (!diffs[targetKey]) throw new Error(`The requested data to restore with key ${targetKey} does not exist`)
+    source = structuredClone(source)
+    const keysById = Object.fromEntries(Object.entries(keys).map(x => [x[1], x[0]]))
+    const propertiesById = Object.fromEntries(Object.entries(properties).map(x => [x[1], x[0]]))
+    const dataByKeys = {} as Record<string, any>
+    for (const [versionKey, { added, changed, removed, removedProps }] of Object.entries(diffs)) {
+      for (const toAdd of added) {
+        dataByKeys[toAdd] = source[toAdd]
+      }
+      for (const toRemove of removed) {
+        delete dataByKeys[toRemove]
+      }
+      for (let i = 0; i < changed.length; i += 2) {
+        const key = changed[i]
+        const change = changed[i + 1]
+        const isOptimizedChange = JsonOptimizer.isOptimizedChangeDiff(change)
+        if (isOptimizedChange) {
+          // apply optimized diff
+          for (let k = 0; k < change.length; k += 2) {
+            const propId = change[k]
+            const newVal = change[k + 1]
+            const prop = propertiesById[propId]
+            // const prop = propId
+            if (prop === undefined) throw new Error(`Property id change is undefined: ${propId}`)
+            dataByKeys[key][prop] = newVal
+          }
+        } else {
+          dataByKeys[key] = change
+        }
+      }
+      for (const [key, removePropsId] of removedProps) {
+        for (const removePropId of removePropsId) {
+          const removeProp = propertiesById[removePropId]
+          delete dataByKeys[key][removeProp]
+        }
+      }
+      if (versionToNumber(versionKey) <= versionToNumber(targetKey)) {
+        break
+      }
+    }
+    if (arrKey) {
+      return Object.values(dataByKeys)
+    } else {
+      return Object.fromEntries(Object.entries(dataByKeys).map(([key, val]) => [keysById[key], val]))
+    }
+  }
+
+  static getByArrKey (item: any, arrKey: string) {
+    return arrKey.split('+').map(x => item[x]).join('+')
+  }
+
+  static resolveDefaults (arr) {
+    if (!Array.isArray(arr)) throw new Error('not an array')
+    const propsValueCount = {} as {
+      [key: string]: {
+        [val: string]: number
+      }
+    }
+    for (const obj of arr) {
+      if (typeof obj !== 'object' || !obj) continue
+      for (const [key, val] of Object.entries(obj)) {
+        const valJson = JSON.stringify(val)
+        propsValueCount[key] ??= {}
+        propsValueCount[key][valJson] ??= 0
+        propsValueCount[key][valJson] += 1
+      }
+    }
+    const defaults = Object.fromEntries(Object.entries(propsValueCount).map(([prop, values]) => {
+      const defaultValue = Object.entries(values).sort(([, count1], [, count2]) => count2 - count1)[0][0]
+      return [prop, defaultValue]
+    }))
+
+    const newData = [] as any[]
+    const noData = {}
+    for (const [i, obj] of arr.entries()) {
+      if (typeof obj !== 'object' || !obj) {
+        newData.push(obj)
+        continue
+      }
+      for (const key of Object.keys(defaults)) {
+        const val = obj[key]
+        if (!val) {
+          noData[key] ??= []
+          noData[key].push(key)
+          continue
+        }
+        if (defaults[key] === JSON.stringify(val)) {
+          delete obj[key]
+        }
+      }
+      newData.push(obj)
+    }
+
+    return {
+      data: newData,
+      defaults
+    }
+  }
+}
+
+const isEqualStructured = (val1, val2) => {
+  return JSON.stringify(val1) === JSON.stringify(val2)
+}

src/react/CreateWorld.tsx

@@ -45,8 +45,8 @@ export default ({ cancelClick, createClick, customizeClick, versions, defaultVer
         placeholder='World name'
       />
       <SelectGameVersion
-        versions={versions.map((obj) => { return { value: obj.version, label: obj.version === defaultVersion ? obj.version + ' (available offline)' : obj.version } })}
-        selected={{ value: defaultVersion, label: defaultVersion + ' (available offline)' }}
+        versions={versions.map((obj) => { return { value: obj.version, label: obj.version } })}
+        selected={{ value: defaultVersion, label: defaultVersion }}
         onChange={(value) => {
           creatingWorldState.version = value ?? defaultVersion
         }}

src/shims/minecraftData.ts

@@ -0,0 +1,92 @@
+import { versionToNumber } from 'prismarine-viewer/viewer/prepare/utils'
+import JsonOptimizer from '../optimizeJson'
+import minecraftInitialDataJson from '../../generated/minecraft-initial-data.json'
+import { toMajorVersion } from '../utils'
+
+const customResolver = () => {
+  const resolver = Promise.withResolvers()
+  let resolvedData
+  return {
+    ...resolver,
+    get resolvedData () {
+      return resolvedData
+    },
+    resolve (data) {
+      resolver.resolve(data)
+      resolvedData = data
+    }
+  }
+}
+
+const optimizedDataResolver = customResolver()
+window._MC_DATA_RESOLVER = optimizedDataResolver
+window._LOAD_MC_DATA = async () => {
+  if (optimizedDataResolver.resolvedData) return
+  optimizedDataResolver.resolve(await import('../../generated/minecraft-data-optimized.json'))
+}
+
+// 30 seconds
+const cacheTtl = 30 * 1000
+const cache = new Map<string, any>()
+const cacheTime = new Map<string, number>()
+const possiblyGetFromCache = (version: string) => {
+  if (minecraftInitialDataJson[version] && !optimizedDataResolver.resolvedData) {
+    return minecraftInitialDataJson[version]
+  }
+  if (cache.has(version)) {
+    return cache.get(version)
+  }
+  const inner = () => {
+    if (!optimizedDataResolver.resolvedData) {
+      throw new Error(`Data for ${version} is not ready yet`)
+    }
+    const dataTypes = Object.keys(optimizedDataResolver.resolvedData)
+    const allRestored = {}
+    for (const dataType of dataTypes) {
+      if (dataType === 'blockCollisionShapes' && versionToNumber(version) >= versionToNumber('1.13')) {
+        const shapes = window.globalGetCollisionShapes?.(version)
+        if (shapes) {
+          allRestored[dataType] = shapes
+          continue
+        }
+      }
+
+      const data = optimizedDataResolver.resolvedData[dataType]
+      if (data.__IS_OPTIMIZED__) {
+        allRestored[dataType] = JsonOptimizer.restoreData(data, version)
+      } else {
+        allRestored[dataType] = data[version] ?? data[toMajorVersion(version)]
+      }
+    }
+    return allRestored
+  }
+  const data = inner()
+  cache.set(version, data)
+  cacheTime.set(version, Date.now())
+  return data
+}
+window.allLoadedMcData = new Proxy({}, {
+  get (t, version: string) {
+    // special properties like $typeof
+    if (version.includes('$')) return
+    // todo enumerate all props
+    return new Proxy({}, {
+      get (target, prop) {
+        return possiblyGetFromCache(version)[prop]
+      },
+    })
+  }
+})
+
+setInterval(() => {
+  const now = Date.now()
+  for (const [version, time] of cacheTime) {
+    if (now - time > cacheTtl) {
+      cache.delete(version)
+      cacheTime.delete(version)
+    }
+  }
+}, 1000)
+
+export const pc = window.allLoadedMcData
+export default { pc }