fix: drop download & deploy size by 1.5x by inlining optimized block collision shapes

package.json

@@ -10,7 +10,7 @@
     "test:cypress": "cypress run",
     "test:e2e": "start-test http-get://localhost:8080 test:cypress",
     "prod-start": "node server.js",
-    "postinstall": "node scripts/gen-texturepack-files.mjs",
+    "postinstall": "node scripts/gen-texturepack-files.mjs && tsx scripts/optimizeBlockCollisions.ts",
     "test-mc-server": "tsx cypress/minecraft-server.mjs",
     "lint": "eslint \"{src,cypress}/**/*.{ts,js,jsx,tsx}\"",
     "storybook": "storybook dev -p 6006",

scripts/optimizeBlockCollisions.ts

@@ -0,0 +1,56 @@
+import fs from 'fs'
+import supportedVersions from '../src/supportedVersions.mjs'
+import { join } from 'path'
+
+type Shape = number[]
+
+interface Shapes {
+  blocks: Record<string, number | number[]>
+  shapes: Record<string, Shape[]>
+}
+
+const shapesMap = new Map<string, number>()
+const processData = (data: Shapes) => {
+  const sizeInput = JSON.stringify(data).length
+  let replaced = 0
+  let shapesToRemove = new Set<number>()
+  for (const [block, shapes] of Object.entries(data.blocks)) {
+    const arr = Array.isArray(shapes)
+    const shapesArr = Array.isArray(shapes) ? shapes : [shapes]
+    for (const [i, id] of shapesArr.entries()) {
+      const resolved = data.shapes[id]
+      const shapesKey = resolved.map(x => x.join(',')).join(';')
+      const existingShapeId = shapesMap.get(shapesKey)
+      // todo-low the size can be optimized even futher by splitting data into shape-parts
+      if (existingShapeId !== undefined && existingShapeId !== id) {
+        // console.log(`duplicate for ${block}: ${existingShapeId}`)
+        if (arr) data.blocks[block][i] = existingShapeId
+        else data.blocks[block] = existingShapeId
+        replaced++
+        shapesToRemove.add(id)
+      }
+      else {
+        shapesMap.set(shapesKey, id)
+      }
+    }
+  }
+  for (const shape of shapesToRemove) {
+    delete data.shapes[shape]
+  }
+  // const sizeOutput = JSON.stringify(data).length
+  // console.log('Saving', replaced, sizeInput / 1024, '->', sizeOutput / 1024, Math.round(sizeInput / sizeOutput), 'x')
+  return data
+}
+
+for (const version of [...supportedVersions].reverse()) {
+  const dataPath = join(require.resolve('minecraft-data'), '../minecraft-data/data/pc', version, 'blockCollisionShapes.json')
+  if (fs.existsSync(dataPath)) {
+    console.log('using blockCollisionShapes of version', version)
+    const data = JSON.parse(fs.readFileSync(dataPath, 'utf8'))
+    processData(data)
+    fs.writeFileSync('./generated/latestBlockCollisionsShapes.json', JSON.stringify(data), 'utf8')
+    break
+  }
+}
+
+// const path = './node_modules/.pnpm/minecraft-data@3.45.0/node_modules/minecraft-data/minecraft-data/data/pc/1.19/blockCollisionShapes.json'

scripts/prepareData.mjs

@@ -3,6 +3,7 @@ import { build } from 'esbuild'
 import { existsSync } from 'node:fs'
 import Module from "node:module"
 import { dirname } from 'node:path'
+import supportedVersions from '../src/supportedVersions.mjs'
 
 if (existsSync('dist/mc-data') && !process.argv.includes('-f')) {
   console.log('using cached prepared data')
@@ -18,25 +19,33 @@ function toMajor (version) {
   return `${a}.${b}`
 }
 
-const ignoredVersionsRegex = /(^0\.30c$)|w|-pre|-rc/
-
 const grouped = {}
 
 for (const [version, data] of Object.entries(dataPaths.pc)) {
-  if (ignoredVersionsRegex.test(version)) continue
+  if (!supportedVersions.includes(version)) continue
   const major = toMajor(version)
   grouped[major] ??= {}
   grouped[major][version] = data
 }
 
+const versionToNumber = (ver) => {
+  const [x, y = '0', z = '0'] = ver.split('.')
+  return +`${x.padStart(2, '0')}${y.padStart(2, '0')}${z.padStart(2, '0')}`
+}
+
 console.log('preparing data')
 console.time('data prepared')
 let builds = []
 for (const [major, versions] of Object.entries(grouped)) {
+  // if (major !== '1.19') continue
   let contents = 'Object.assign(window.mcData, {\n'
   for (const [version, dataSet] of Object.entries(versions)) {
     contents += `    '${version}': {\n`
     for (const [dataType, dataPath] of Object.entries(dataSet)) {
+      if (dataType === 'blockCollisionShapes' && versionToNumber(version) >= versionToNumber('1.13')) {
+        contents += `      get ${dataType} () { return window.globalGetCollisionShapes?.("${version}") },\n`
+        continue
+      }
       const loc = `minecraft-data/data/${dataPath}/`
       contents += `      get ${dataType} () { return require("./${loc}${dataType}.json") },\n`
     }
@@ -54,7 +63,9 @@ for (const [major, versions] of Object.entries(grouped)) {
       sourcefile: `mcData${major}.js`,
       loader: 'js',
     },
+    metafile: true,
   })
+  // require('fs').writeFileSync('dist/mc-data/metafile.json', JSON.stringify(promise.metafile), 'utf8')
   builds.push(promise)
 }
 await Promise.all(builds)

src/getCollisionShapes.ts

@@ -0,0 +1,13 @@
+import { adoptBlockOrItemNamesFromLatest } from 'flying-squid/src/blockRenames'
+import collisionShapesInit from '../generated/latestBlockCollisionsShapes.json'
+
+// defining globally to be used in loaded data, not sure of better workaround
+window.globalGetCollisionShapes = (version) => {
+  // todo use the same in resourcepack
+  const renamedBlocks = adoptBlockOrItemNamesFromLatest('blocks', version, Object.keys(collisionShapesInit.blocks))
+  const collisionShapes = {
+    ...collisionShapesInit,
+    blocks: Object.fromEntries(Object.entries(collisionShapesInit.blocks).map(([, shape], i) => [renamedBlocks[i], shape]))
+  }
+  return collisionShapes
+}

src/index.ts

@@ -4,6 +4,7 @@ import './styles.css'
 import './globals'
 import 'iconify-icon'
 import './chat'
+import './getCollisionShapes'
 import { onGameLoad } from './playerWindows'
 
 import './menus/components/button'
@@ -21,6 +22,7 @@ import './menus/play_screen'
 import './menus/pause_screen'
 import './menus/keybinds_screen'
 import { initWithRenderer, statsEnd, statsStart } from './topRightStats'
+import PrismarineBlock from 'prismarine-block'
 
 import { options, watchValue } from './optionsStorage'
 import './reactUi.jsx'
@@ -41,7 +43,7 @@ import { Vec3 } from 'vec3'
 import worldInteractions from './worldInteractions'
 
 import * as THREE from 'three'
-import { versionsByMinecraftVersion } from 'minecraft-data'
+import MinecraftData, { versionsByMinecraftVersion } from 'minecraft-data'
 
 import { initVR } from './vr'
 import {
@@ -250,7 +252,7 @@ async function connect (connectOptions: {
   const p2pMultiplayer = !!connectOptions.peerId
   miscUiState.singleplayer = singleplayer
   miscUiState.flyingSquid = singleplayer || p2pMultiplayer
-  const { renderDistance, maxMultiplayerRenderDistance = renderDistance } = options
+  const { renderDistance: renderDistanceSingleplayer, maxMultiplayerRenderDistance = renderDistanceSingleplayer } = options
   const server = cleanConnectIp(connectOptions.server, '25565')
   const proxy = cleanConnectIp(connectOptions.proxy, undefined)
   const { username, password } = connectOptions
@@ -266,7 +268,7 @@ async function connect (connectOptions: {
     if (ended) return
     ended = true
     viewer.resetAll()
-    window.localServer = undefined
+    localServer = window.localServer = window.server = undefined
 
     postRenderFrameFn = () => { }
     if (bot) {
@@ -330,6 +332,7 @@ async function connect (connectOptions: {
     net['setProxy']({ hostname: proxy.host, port: proxy.port })
   }
 
+  const renderDistance = singleplayer ? renderDistanceSingleplayer : Math.min(renderDistanceSingleplayer, maxMultiplayerRenderDistance!)
   let localServer
   try {
     const serverOptions = _.defaultsDeep({}, connectOptions.serverOverrides ?? {}, options.localServerOptions, defaultServerOptions)
@@ -367,7 +370,7 @@ async function connect (connectOptions: {
       // flying-squid: 'login' -> player.login -> now sends 'login' event to the client (handled in many plugins in mineflayer) -> then 'update_health' is sent which emits 'spawn' in mineflayer
 
       setLoadingScreenStatus('Starting local server')
-      localServer = window.localServer = startLocalServer(serverOptions)
+      localServer = window.localServer = window.server = startLocalServer(serverOptions)
       // todo need just to call quit if started
       // loadingScreen.maybeRecoverable = false
       // init world, todo: do it for any async plugins
@@ -411,7 +414,7 @@ async function connect (connectOptions: {
       } : {},
       username,
       password,
-      viewDistance: 'tiny',
+      viewDistance: renderDistance,
       checkTimeoutInterval: 240 * 1000,
       noPongTimeout: 240 * 1000,
       closeTimeout: 240 * 1000,
@@ -496,6 +499,12 @@ async function connect (connectOptions: {
 
   onBotCreate()
 
+  const mcData = MinecraftData(bot.version)
+  window.PrismarineBlock = PrismarineBlock(mcData.version.minecraftVersion!)
+  window.loadedData = mcData
+  window.Vec3 = Vec3
+  window.pathfinder = pathfinder
+
   bot.once('login', () => {
     if (!connectOptions.server) return
     // server is ok, add it to the history
@@ -510,26 +519,19 @@ async function connect (connectOptions: {
   bot.once('health', () => {
     miscUiState.gameLoaded = true
     if (p2pConnectTimeout) clearTimeout(p2pConnectTimeout)
-    const mcData = require('minecraft-data')(bot.version)
 
     setLoadingScreenStatus('Placing blocks (starting viewer)')
 
     console.log('bot spawned - starting viewer')
 
-    const { version } = bot
-
     const center = bot.entity.position
 
-    const worldView = window.worldView = new WorldDataEmitter(bot.world, singleplayer ? renderDistance : Math.min(renderDistance, maxMultiplayerRenderDistance!), center)
-    setRenderDistance()
+    const worldView = window.worldView = new WorldDataEmitter(bot.world, renderDistance, center)
 
     bot.on('physicsTick', () => updateCursor())
 
     const debugMenu = hud.shadowRoot.querySelector('#debug-overlay')
 
-    window.loadedData = mcData
-    window.Vec3 = Vec3
-    window.pathfinder = pathfinder
     window.debugMenu = debugMenu
 
     void initVR(bot, renderer, viewer)

src/menus/play_screen.js

@@ -1,13 +1,11 @@
 //@ts-check
 const { LitElement, html, css } = require('lit')
-const mineflayer = require('mineflayer')
 const viewerSupportedVersions = require('prismarine-viewer/viewer/supportedVersions.json')
-const { versionsByMinecraftVersion } = require('minecraft-data')
 const { hideCurrentModal, miscUiState } = require('../globalState')
+const { default: supportedVersions } = require('../supportedVersions.mjs')
 const { commonCss } = require('./components/common')
 
 const fullySupporedVersions = viewerSupportedVersions
-const partiallySupportVersions = mineflayer.supportedVersions
 
 class PlayScreen extends LitElement {
   static get styles () {
@@ -176,7 +174,7 @@ class PlayScreen extends LitElement {
             pmui-id="botversion"
             pmui-value="${this.version}"
             pmui-inputmode="decimal"
-            state="${this.version && (fullySupporedVersions.includes(/** @type {any} */(this.version)) ? '' : Object.keys(versionsByMinecraftVersion.pc).includes(this.version) ? 'warning' : 'invalid')}"
+            state="${this.version && (fullySupporedVersions.includes(/** @type {any} */(this.version)) ? '' : supportedVersions.includes(this.version) ? 'warning' : 'invalid')}"
             .autocompleteValues=${fullySupporedVersions}
             @input=${e => { this.version = e.target.value = e.target.value.replaceAll(',', '.') }}
           ></pmui-editbox>

src/react/CreateWorldProvider.tsx

@@ -9,8 +9,7 @@ export default () => {
   const activeCreate = useIsModalActive('create-world')
   const activeCustomize = useIsModalActive('customize-world')
   if (activeCreate) {
-    const ignoredVersionsRegex = /(^0\.30c$)|w|-pre|-rc/
-    const versions = supportedVersions.filter(x => !ignoredVersionsRegex.test(x)).map(x => {
+    const versions = supportedVersions.map(x => {
       return {
         version: x,
         label: x === defaultLocalServerOptions.version ? `${x} (available offline)` : x

src/supportedVersions.mjs

@@ -0,0 +1,5 @@
+import { supportedVersions } from 'minecraft-data'
+
+const ignoredVersionsRegex = /(^0\.30c$)|w|-pre|-rc/
+
+export default supportedVersions.pc.filter(v => !ignoredVersionsRegex.test(v))

src/topRightStats.ts

@@ -21,6 +21,7 @@ const addStat = (dom, size = 80) => {
   dom.style.right = `${total}px`
   dom.style.width = '80px'
   dom.style.zIndex = 1000
+  dom.style.opacity = '0.75'
   document.body.appendChild(dom)
   total += size
 }

src/utils.ts

@@ -169,11 +169,11 @@ export const toMajorVersion = (version) => {
 let prevRenderDistance = options.renderDistance
 export const setRenderDistance = () => {
   assertDefined(worldView)
-  worldView.viewDistance = options.renderDistance
-  if (localServer) {
-    localServer.players[0].emit('playerChangeRenderDistance', options.renderDistance)
-  }
-  prevRenderDistance = options.renderDistance
+  const { renderDistance } = options
+  bot.setSettings({
+    viewDistance: renderDistance
+  })
+  prevRenderDistance = renderDistance
 }
 export const reloadChunks = async () => {
   if (!worldView) return