feat: new experimental chunk loading logic by forcing into spiral queue

feat: add highly experimental logic to try to self-restore any issues with chunks loading by automating f3+a action. write debug info into chat for now. can be disabled
feat: rework chunks debug screen showing actually useful information now

renderer/playground/shared.ts

@@ -65,7 +65,7 @@ function getAllMethods (obj) {
   return [...methods] as string[]
 }
 
-export const delayedIterator = async <T> (arr: T[], delay: number, exec: (item: T, index: number) => void, chunkSize = 1) => {
+export const delayedIterator = async <T> (arr: T[], delay: number, exec: (item: T, index: number) => Promise<void>, chunkSize = 1) => {
   // if delay is 0 then don't use setTimeout
   for (let i = 0; i < arr.length; i += chunkSize) {
     if (delay) {
@@ -74,6 +74,6 @@ export const delayedIterator = async <T> (arr: T[], delay: number, exec: (item:
         setTimeout(resolve, delay)
       })
     }
-    exec(arr[i], i)
+    await exec(arr[i], i)
   }
 }

renderer/viewer/lib/worldDataEmitter.ts

@@ -35,7 +35,15 @@ export class WorldDataEmitterWorker extends (EventEmitter as new () => TypedEmit
 }
 
 export class WorldDataEmitter extends (EventEmitter as new () => TypedEmitter<WorldDataEmitterEvents>) {
+  spiralNumber = 0
+  gotPanicLastTime = false
+  panicChunksReload = () => {}
   loadedChunks: Record<ChunkPosKey, boolean>
+  private inLoading = false
+  private chunkReceiveTimes: number[] = []
+  private lastChunkReceiveTime = 0
+  public lastChunkReceiveTimeAvg = 0
+  private panicTimeout?: NodeJS.Timeout
   readonly lastPos: Vec3
   private eventListeners: Record<string, any> = {}
   private readonly emitter: WorldDataEmitter
@@ -133,12 +141,19 @@ export class WorldDataEmitter extends (EventEmitter as new () => TypedEmitter<Wo
         this.emitter.emit('entity', { id: e.id, delete: true })
       },
       chunkColumnLoad: (pos: Vec3) => {
+        const now = performance.now()
+        if (this.lastChunkReceiveTime) {
+          this.chunkReceiveTimes.push(now - this.lastChunkReceiveTime)
+        }
+        this.lastChunkReceiveTime = now
+
         if (this.waitingSpiralChunksLoad[`${pos.x},${pos.z}`]) {
           this.waitingSpiralChunksLoad[`${pos.x},${pos.z}`](true)
           delete this.waitingSpiralChunksLoad[`${pos.x},${pos.z}`]
         } else if (this.loadedChunks[`${pos.x},${pos.z}`]) {
           void this.loadChunk(pos, false, 'Received another chunkColumnLoad event while already loaded')
         }
+        this.chunkProgress()
       },
       chunkColumnUnload: (pos: Vec3) => {
         this.unloadChunk(pos)
@@ -219,33 +234,59 @@ export class WorldDataEmitter extends (EventEmitter as new () => TypedEmitter<Wo
     const positions = generateSpiralMatrix(this.viewDistance).map(([x, z]) => new Vec3((botX + x) * 16, 0, (botZ + z) * 16))
 
     this.lastPos.update(pos)
-    await this._loadChunks(positions)
+    await this._loadChunks(positions, pos)
   }
 
-  async _loadChunks (positions: Vec3[], sliceSize = 5) {
+  chunkProgress () {
+    if (this.panicTimeout) clearTimeout(this.panicTimeout)
+    if (this.chunkReceiveTimes.length >= 5) {
+      const avgReceiveTime = this.chunkReceiveTimes.reduce((a, b) => a + b, 0) / this.chunkReceiveTimes.length
+      this.lastChunkReceiveTimeAvg = avgReceiveTime
+      const timeoutDelay = avgReceiveTime * 2 + 1000 // 2x average + 1 second
+
+      // Clear any existing timeout
+      if (this.panicTimeout) clearTimeout(this.panicTimeout)
+
+      // Set new timeout for panic reload
+      this.panicTimeout = setTimeout(() => {
+        if (!this.gotPanicLastTime && this.inLoading) {
+          console.warn('Chunk loading seems stuck, triggering panic reload')
+          this.gotPanicLastTime = true
+          this.panicChunksReload()
+        }
+      }, timeoutDelay)
+    }
+  }
+
+  async _loadChunks (positions: Vec3[], centerPos: Vec3) {
+    this.spiralNumber++
+    const { spiralNumber } = this
     // stop loading previous chunks
     for (const pos of Object.keys(this.waitingSpiralChunksLoad)) {
       this.waitingSpiralChunksLoad[pos](false)
       delete this.waitingSpiralChunksLoad[pos]
     }
 
-    const promises = [] as Array<Promise<void>>
     let continueLoading = true
+    this.inLoading = true
     await delayedIterator(positions, this.addWaitTime, async (pos) => {
-      const promise = (async () => {
-        if (!continueLoading || this.loadedChunks[`${pos.x},${pos.z}`]) return
+      if (!continueLoading || this.loadedChunks[`${pos.x},${pos.z}`]) return
 
-        if (!this.world.getColumnAt(pos)) {
-          continueLoading = await new Promise<boolean>(resolve => {
-            this.waitingSpiralChunksLoad[`${pos.x},${pos.z}`] = resolve
-          })
-        }
-        if (!continueLoading) return
-        await this.loadChunk(pos)
-      })()
-      promises.push(promise)
+      // Wait for chunk to be available from server
+      if (!this.world.getColumnAt(pos)) {
+        continueLoading = await new Promise<boolean>(resolve => {
+          this.waitingSpiralChunksLoad[`${pos.x},${pos.z}`] = resolve
+        })
+      }
+      if (!continueLoading) return
+      await this.loadChunk(pos, undefined, `spiral ${spiralNumber} from ${centerPos.x},${centerPos.z}`)
+      this.chunkProgress()
     })
-    await Promise.all(promises)
+    if (this.panicTimeout) clearTimeout(this.panicTimeout)
+    this.inLoading = false
+    this.gotPanicLastTime = false
+    this.chunkReceiveTimes = []
+    this.lastChunkReceiveTime = 0
   }
 
   readdDebug () {
@@ -350,7 +391,7 @@ export class WorldDataEmitter extends (EventEmitter as new () => TypedEmitter<Wo
         return undefined!
       }).filter(a => !!a)
       this.lastPos.update(pos)
-      void this._loadChunks(positions)
+      void this._loadChunks(positions, pos)
     } else {
       this.emitter.emit('chunkPosUpdate', { pos }) // todo-low
       this.lastPos.update(pos)

renderer/viewer/three/entities.ts

@@ -762,7 +762,6 @@ export class Entities {
       }, {
         faceCamera,
         use3D: !faceCamera, // Only use 3D for non-camera-facing items
-        depth: 0.03
       })
 
       let SCALE = 1

renderer/viewer/three/itemMesh.ts

@@ -1,7 +1,7 @@
 import * as THREE from 'three'
 
 export interface Create3DItemMeshOptions {
-  depth?: number
+  depth: number
   pixelSize?: number
 }
 
@@ -17,9 +17,9 @@ export interface Create3DItemMeshResult {
  */
 export function create3DItemMesh (
   canvas: HTMLCanvasElement,
-  options: Create3DItemMeshOptions = {}
+  options: Create3DItemMeshOptions
 ): Create3DItemMeshResult {
-  const { depth = 0.03, pixelSize } = options
+  const { depth, pixelSize } = options
 
   // Validate canvas dimensions
   if (canvas.width <= 0 || canvas.height <= 0) {
@@ -287,7 +287,7 @@ export function createItemMesh (
     depth?: number
   } = {}
 ): ItemMeshResult {
-  const { faceCamera = false, use3D = true, depth = 0.03 } = options
+  const { faceCamera = false, use3D = true, depth = 0.04 } = options
   const { u, v, sizeX, sizeY } = textureInfo
 
   if (faceCamera) {
@@ -403,7 +403,7 @@ export function createItemMesh (
  */
 export function createItemMeshFromCanvas (
   canvas: HTMLCanvasElement,
-  options: Create3DItemMeshOptions = {}
+  options: Create3DItemMeshOptions
 ): THREE.Mesh {
   const { geometry } = create3DItemMesh(canvas, options)
 

src/appViewer.ts

@@ -17,6 +17,8 @@ import { options } from './optionsStorage'
 import { ResourcesManager, ResourcesManagerTransferred } from './resourcesManager'
 import { watchOptionsAfterWorldViewInit } from './watchOptions'
 import { loadMinecraftData } from './connect'
+import { reloadChunks } from './utils'
+import { displayClientChat } from './botUtils'
 
 export interface RendererReactiveState {
   world: {
@@ -197,6 +199,11 @@ export class AppViewer {
     this.currentDisplay = 'world'
     const startPosition = bot.entity?.position ?? new Vec3(0, 64, 0)
     this.worldView = new WorldDataEmitter(world, renderDistance, startPosition)
+    this.worldView.panicChunksReload = () => {
+      if (!options.experimentalClientSelfReload) return
+      displayClientChat(`[client] client panicked due to too long loading time. Soft reloading chunks...`)
+      void reloadChunks()
+    }
     window.worldView = this.worldView
     watchOptionsAfterWorldViewInit(this.worldView)
     this.appConfigUdpate()

src/defaultOptions.ts

@@ -84,6 +84,7 @@ export const defaultOptions = {
     gameMode: 1
   } as any,
   preferLoadReadonly: false,
+  experimentalClientSelfReload: true,
   disableLoadPrompts: false,
   guestUsername: 'guest',
   askGuestName: true,

src/react/ChunksDebug.tsx

@@ -98,13 +98,16 @@ export default ({
                 cursor: chunk ? 'pointer' : 'default',
                 position: 'relative',
                 width: `${tileSize}px`,
+                flexDirection: 'column',
                 height: `${tileSize}px`,
+                padding: 1,
                 // pre-wrap
                 whiteSpace: 'pre',
               }}
             >
               {relX}, {relZ}{'\n'}
-              {chunk?.lines.join('\n')}
+              {chunk?.lines[0]}{'\n'}
+              <span style={{ fontSize: `${fontSize * 0.8}px` }}>{chunk?.lines[1]}</span>
             </div>
           )
         })}

src/react/ChunksDebugScreen.tsx

@@ -2,6 +2,8 @@ import { useEffect, useState } from 'react'
 import { useUtilsEffect } from '@zardoy/react-util'
 import { WorldRendererCommon } from 'renderer/viewer/lib/worldrendererCommon'
 import { WorldRendererThree } from 'renderer/viewer/three/worldrendererThree'
+import { Vec3 } from 'vec3'
+import { generateSpiralMatrix } from 'flying-squid/dist/utils'
 import Screen from './Screen'
 import ChunksDebug, { ChunkDebug } from './ChunksDebug'
 import { useIsModalActive } from './utilsApp'
@@ -12,6 +14,10 @@ const Inner = () => {
   const [update, setUpdate] = useState(0)
 
   useUtilsEffect(({ interval }) => {
+    const up = () => {
+      // setUpdate(u => u + 1)
+    }
+    bot.on('chunkColumnLoad', up)
     interval(
       500,
       () => {
@@ -20,15 +26,46 @@ const Inner = () => {
         setUpdate(u => u + 1)
       }
     )
+    return () => {
+      bot.removeListener('chunkColumnLoad', up)
+    }
   }, [])
 
+  // Track first load time for all chunks
+  const allLoadTimes = Object.values(worldView!.debugChunksInfo)
+    .map(chunk => chunk?.loads[0]?.time ?? Infinity)
+    .filter(time => time !== Infinity)
+    .sort((a, b) => a - b)
+
+  const allSpiralChunks = Object.fromEntries(generateSpiralMatrix(worldView!.viewDistance).map(pos => [`${pos[0]},${pos[1]}`, pos]))
+
   const mapChunk = (key: string, state: ChunkDebug['state']): ChunkDebug => {
+    const x = Number(key.split(',')[0])
+    const z = Number(key.split(',')[1])
+    const chunkX = Math.floor(x / 16)
+    const chunkZ = Math.floor(z / 16)
+
+    delete allSpiralChunks[`${chunkX},${chunkZ}`]
     const chunk = worldView!.debugChunksInfo[key]
+    const firstLoadTime = chunk?.loads[0]?.time
+    const loadIndex = firstLoadTime ? allLoadTimes.indexOf(firstLoadTime) + 1 : 0
+    // const timeSinceFirstLoad = firstLoadTime ? firstLoadTime - allLoadTimes[0] : 0
+    const timeSinceFirstLoad = firstLoadTime ? firstLoadTime - allLoadTimes[0] : 0
+    let line = ''
+    let line2 = ''
+    if (loadIndex) {
+      line = `${loadIndex}`
+      line2 = `${timeSinceFirstLoad}ms`
+    }
+    if (chunk?.loads.length > 1) {
+      line += ` - ${chunk.loads.length}`
+    }
+
     return {
-      x: Number(key.split(',')[0]),
-      z: Number(key.split(',')[1]),
+      x,
+      z,
       state,
-      lines: [String(chunk?.loads.length ?? 0)],
+      lines: [line, line2],
       sidebarLines: [
         `loads: ${chunk?.loads?.map(l => `${l.reason} ${l.dataLength} ${l.time}`).join('\n')}`,
         // `blockUpdates: ${chunk.blockUpdates}`,
@@ -55,14 +92,22 @@ const Inner = () => {
 
   const chunksDone = Object.keys(world.finishedChunks).map(key => mapChunk(key, 'done'))
 
+
+  const chunksWaitingOrder = Object.values(allSpiralChunks).map(([x, z]) => {
+    const pos = new Vec3(x * 16, 0, z * 16)
+    if (bot.world.getColumnAt(pos) === null) return null
+    return mapChunk(`${pos.x},${pos.z}`, 'order-queued')
+  }).filter(a => !!a)
+
   const allChunks = [
     ...chunksWaitingServer,
     ...chunksWaitingClient,
     ...clientProcessingChunks,
     ...chunksDone,
     ...chunksDoneEmpty,
+    ...chunksWaitingOrder,
   ]
-  return <Screen title="Chunks Debug">
+  return <Screen title={`Chunks Debug (avg: ${worldView!.lastChunkReceiveTimeAvg.toFixed(1)}ms)`}>
     <ChunksDebug
       chunks={allChunks}
       playerChunk={{