inventory support with sprites (only multiplayer fully supported for now)! true block rendering!

add iphone x notch workarounds

package.json

@@ -92,7 +92,8 @@
     "webpack-dev-middleware": "^6.1.1",
     "webpack-dev-server": "^4.15.1",
     "webpack-merge": "^5.9.0",
-    "workbox-webpack-plugin": "^6.6.0"
+    "workbox-webpack-plugin": "^6.6.0",
+    "minecraft-inventory-gui": "github:zardoy/minecraft-inventory-gui#next"
   },
   "pnpm": {
     "overrides": {

src/globalState.js

@@ -102,16 +102,17 @@ export const showContextmenu = (/** @type {ContextMenuItem[]} */items, { clientX
 
 // ---
 
-export const isGameActive = (foregroundCheck) => {
-  if (foregroundCheck && activeModalStack.length) return false
-  return document.getElementById('hud').style.display !== 'none'
-}
-
 export const miscUiState = proxy({
   currentTouch: null,
-  singleplayer: false
+  singleplayer: false,
+  gameLoaded: false,
 })
 
+export const isGameActive = (foregroundCheck) => {
+  if (foregroundCheck && activeModalStack.length) return false
+  return miscUiState.gameLoaded
+}
+
 window.miscUiState = miscUiState
 
 // state that is not possible to get via bot and in-game specific

src/globals.js

@@ -0,0 +1,5 @@
+//@ts-nocheck
+
+window.bot = undefined
+window.THREE = undefined
+window.singlePlayerServer = undefined

src/index.js

@@ -8,6 +8,9 @@
 require('./styles.css')
 require('iconify-icon')
 require('./chat')
+require('./inventory')
+//@ts-ignore
+require('./globals.js')
 
 // workaround for mineflayer
 process.versions.node = '18.0.0'
@@ -172,6 +175,8 @@ const optionsScrn = document.getElementById('options-screen')
 const pauseMenu = document.getElementById('pause-screen')
 
 function setLoadingScreenStatus (status, isError = false) {
+  // todo update in component instead
+  miscUiState.gameLoaded = false
   showModal(loadingScreen)
   if (loadingScreen.hasError) return
   loadingScreen.hasError = isError
@@ -348,6 +353,11 @@ async function connect (connectOptions) {
 
   const errorAbortController = new AbortController()
   window.addEventListener('unhandledrejection', (e) => {
+    if (e.reason.name === 'ServerPluginLoadFailure') {
+      if (confirm(`Failed to load server plugin ${e.reason.pluginName} (invoking ${e.reason.pluginMethod}). Continue?`)) {
+        return
+      }
+    }
     handleError(e.reason)
   }, {
     signal: errorAbortController.signal
@@ -433,6 +443,7 @@ async function connect (connectOptions) {
   })
 
   bot.once('spawn', () => {
+    miscUiState.gameLoaded = true
     // todo display notification if not critical
     const mcData = require('minecraft-data')(bot.version)
 

src/inventory.ts

@@ -0,0 +1,136 @@
+import { subscribe } from 'valtio'
+import { activeModalStack, hideCurrentModal, miscUiState } from './globalState'
+import { showInventory } from 'minecraft-inventory-gui/web/ext.mjs'
+import InventoryGui from 'minecraft-assets/minecraft-assets/data/1.17.1/gui/container/inventory.png'
+import Dirt from 'minecraft-assets/minecraft-assets/data/1.17.1/blocks/dirt.png'
+import { subscribeKey } from 'valtio/utils'
+import MinecraftData from 'minecraft-data'
+import invspriteJson from './invsprite.json'
+import { getVersion } from 'prismarine-viewer/viewer/lib/version'
+
+const loadedImages = new Map<string, HTMLImageElement>()
+let blockStates: Record<string, null | { variants: Record<string, { model: { textures: { up: { u, v, su, sv } } } }[]> }>
+let lastInventory
+let mcData
+let version
+
+subscribeKey(miscUiState, 'gameLoaded', async () => {
+  if (!miscUiState.gameLoaded) {
+    // loadedBlocksAtlas = null
+    return
+  }
+
+  // on game load
+  version = getVersion(bot.version)
+  blockStates = await fetch(`blocksStates/${version}.json`).then(res => res.json())
+  getImage({ path: 'blocks', } as any)
+  getImage({ path: 'invsprite', } as any)
+  mcData = MinecraftData(version)
+})
+
+const findBlockStateTexturesAtlas = (name) => {
+  const vars = blockStates[name]?.variants
+  if (!vars) return
+  const firstVar = Object.values(vars)[0]
+  if (!firstVar || !Array.isArray(firstVar)) return
+  return firstVar[0]?.model.textures
+}
+
+const getBlockData = (name) => {
+  const blocksImg = loadedImages.get('blocks')
+  if (!blocksImg || !blocksImg.width) return
+
+  const data = findBlockStateTexturesAtlas(name)
+  if (!data) return
+
+  const getSpriteBlockSide = (side) => {
+    const d = data[side]
+    const spriteSide = [d.u * blocksImg.width, d.v * blocksImg.height, d.su * blocksImg.width, d.sv * blocksImg.height]
+    const blockSideData = {
+      slice: spriteSide,
+      path: 'blocks'
+    }
+    return blockSideData
+  }
+
+  return {
+    top: getSpriteBlockSide('up'),
+    left: getSpriteBlockSide('east'),
+    right: getSpriteBlockSide('north'),
+  }
+}
+
+const getItemSlice = (name) => {
+  const invspriteImg = loadedImages.get('invsprite')
+  if (!invspriteImg || !invspriteImg.width) return
+
+  const { x, y } = invspriteJson[name]
+  const sprite = [x, y, 32, 32]
+  return sprite
+}
+
+const getImageSrc = (path) => {
+  switch (path) {
+    case 'gui/container/inventory': return InventoryGui
+    case 'blocks': return `textures/${version}.png`
+    case 'invsprite': return `invsprite.png`
+  }
+  return Dirt
+}
+
+const getImage = ({ path, texture, blockData }) => {
+  const loadPath = blockData ? 'blocks' : path ?? texture
+  if (!loadedImages.has(loadPath)) {
+    const image = new Image()
+    // image.onload(() => {})
+    image.src = getImageSrc(loadPath)
+    loadedImages.set(loadPath, image)
+  }
+  return loadedImages.get(loadPath)
+}
+
+const upInventory = () => {
+  // inv.pwindow.inv.slots[2].displayName = 'test'
+  // inv.pwindow.inv.slots[2].blockData = getBlockData('dirt')
+  const customSlots = bot.inventory.slots.map(slot => {
+    if (!slot) return
+    // const itemName = slot.name
+    // const isItem = mcData.itemsByName[itemName]
+
+    // try get block data first, but ideally we should also have atlas from atlas/ folder
+    const blockData = getBlockData(slot.name)
+    if (blockData) {
+      slot['texture'] = 'blocks'
+      slot['blockData'] = blockData
+    } else {
+      slot['texture'] = 'invsprite'
+      slot['scale'] = 0.5
+      slot['slice'] = getItemSlice(slot.name)
+    }
+
+    return slot
+  })
+  lastInventory.pwindow.setSlots(customSlots)
+}
+
+subscribe(activeModalStack, () => {
+  const inventoryOpened = activeModalStack.slice(-1)[0]?.reactType === 'inventory'
+  if (inventoryOpened) {
+    const inv = showInventory(undefined, getImage, {}, bot)
+    inv.canvas.style.zIndex = 10
+    inv.canvas.style.position = 'fixed'
+    inv.canvas.style.inset = '0'
+    // todo scaling
+    inv.canvasManager.setScale(window.innerHeight < 480 ? 2 : window.innerHeight < 700 ? 3 : 4)
+    inv.canvasManager.onClose = () => {
+      hideCurrentModal()
+      inv.canvasManager.destroy()
+    }
+
+    lastInventory = inv
+    upInventory()
+  } else if (lastInventory) {
+    lastInventory.destroy()
+    lastInventory = null
+  }
+})

src/menus/components/common.js

@@ -48,6 +48,14 @@ function isMobile () {
   return window.matchMedia('(pointer: coarse)').matches
 }
 
+// todo there are better workarounds and proper way to detect notch
+/** @returns {boolean} */
+function isProbablyIphone () {
+  if (!isMobile()) return false
+  const smallest = window.innerWidth < window.innerHeight ? window.innerWidth : window.innerHeight
+  return smallest < 600
+}
+
 /**
  * @param {string} url
  */
@@ -56,6 +64,7 @@ function openURL (url) {
 }
 
 export {
+  isProbablyIphone,
   commonCss,
   isMobile,
   openURL,

src/menus/components/hotbar.js

@@ -1,15 +1,17 @@
 const { LitElement, html, css, unsafeCSS } = require('lit')
 const invsprite = require('../../invsprite.json')
-const { isGameActive } = require('../../globalState')
+const { isGameActive, miscUiState, showModal } = require('../../globalState')
 
 const widgetsTexture = require('minecraft-assets/minecraft-assets/data/1.16.4/gui/widgets.png')
+const { subscribeKey } = require('valtio/utils')
+const { isProbablyIphone } = require('./common')
 
 class Hotbar extends LitElement {
   static get styles () {
     return css`
       .hotbar {
         position: absolute;
-        bottom: 0;
+        bottom: ${unsafeCSS(isProbablyIphone() ? '40px' : '0')};
         left: 50%;
         transform: translate(-50%);
         width: 182px;
@@ -84,6 +86,16 @@ class Hotbar extends LitElement {
         transition: visibility 0s, opacity 1s linear;
         transition-delay: 2s;
       }
+
+      .hotbar-more {
+        display:flex;
+        justify-content: center;
+        border: 1px solid white;
+      }
+      .hotbar-more::before {
+        content: '...';
+        margin-top: -1px;
+      }
     `
   }
 
@@ -97,6 +109,9 @@ class Hotbar extends LitElement {
 
   constructor () {
     super()
+    subscribeKey(miscUiState, 'currentTouch', () => {
+      this.requestUpdate()
+    })
     this.activeItemName = ''
   }
 
@@ -209,6 +224,10 @@ class Hotbar extends LitElement {
             <div class="item-icon"></div>
             <span class="item-stack"></span>
           </div>
+          <div class="hotbar-item hotbar-more" ?hidden=${!miscUiState.currentTouch} @click=${() => {
+        showModal({ reactType: 'inventory', })
+      }}>
+          </div>
         </div>
       </div>
     `

src/menus/loading_or_error_screen.js

@@ -78,6 +78,7 @@ class LoadingErrorScreen extends LitElement {
       ${this.hasError
         ? html`<div class="error-buttons"><pmui-button .hidden=${!this.maybeRecoverable} pmui-width="200px" pmui-label="Back" @pmui-click=${() => {
           this.hasError = false
+          miscUiState.gameLoaded = false
           if (activeModalStacks['main-menu']) {
             replaceActiveModalStack('main-menu')
           } else {

src/menus/pause_screen.js

@@ -1,7 +1,7 @@
 //@ts-check
 const { LitElement, html, css } = require('lit')
 const { openURL } = require('./components/common')
-const { hideCurrentModal, showModal } = require('../globalState')
+const { hideCurrentModal, showModal, miscUiState } = require('../globalState')
 const { fsState } = require('../loadFolder')
 const { subscribe } = require('valtio')
 const { saveWorld } = require('../builtinCommands')
@@ -76,6 +76,7 @@ class PauseScreen extends LitElement {
           singlePlayerServer.quit()
         }
         bot._client.emit('end')
+        miscUiState.gameLoaded = false
       }}></pmui-button>
       </main>
     `

src/reactUi.jsx

@@ -7,12 +7,13 @@ import { activeModalStack, hideCurrentModal, isGameActive } from './globalState'
 import { useEffect, useState } from 'react'
 import { useProxy } from 'valtio/utils'
 import useTypedEventListener from 'use-typed-event-listener'
+import { isProbablyIphone } from './menus/components/common'
 
 // todo
 useInterfaceState.setState({
     isFlying: false,
     uiCustomization: {
-        touchButtonSize: 40,
+        touchButtonSize: isProbablyIphone() ? 55 : 40,
     },
     updateCoord: ([coord, state]) => {
         const coordToAction = [
@@ -106,21 +107,23 @@ function InventoryWrapper() {
 
     if (!isInventoryOpen) return null
 
-    return <div className={css`
-            position: fixed;
-            width: 100%;
-            height: 100%;
-            background: rgba(0, 0, 0, 0.5);
+    return null
 
-            & > div {
-                scale: 0.6;
-                background: transparent !important;
-            }
-        `}>
-        <InventoryNew slots={slots} action={(oldSlot, newSlotIndex) => {
-            bot.moveSlotItem(oldSlot, newSlotIndex)
-        } } />
-    </div>
+    // return <div className={css`
+    //         position: fixed;
+    //         width: 100%;
+    //         height: 100%;
+    //         background: rgba(0, 0, 0, 0.5);
+
+    //         & > div {
+    //             scale: 0.6;
+    //             background: transparent !important;
+    //         }
+    //     `}>
+    //     <InventoryNew slots={slots} action={(oldSlot, newSlotIndex) => {
+    //         bot.moveSlotItem(oldSlot, newSlotIndex)
+    //     } } />
+    // </div>
 }
 
 const App = () => {

src/styles.css

@@ -92,12 +92,10 @@ canvas {
   font-family: minecraft, mojangles, monospace;
 }
 
-/* todo I guess the best solution would be to render it on canvas */
-.BlockModel {
-  scale: 0.5;
-  width: 183%;
-  height: 190%;
-  transform: translate(-50%, -50%);
+/* todo move this fix to lib */
+.TouchMovementArea {
+  grid-template-columns: "c ."
+  "d .";
 }
 
 @media only screen and (max-width: 971px) {

tsconfig.json

@@ -7,7 +7,8 @@
         "jsx": "react-jsx",
         "allowSyntheticDefaultImports": true,
         "noEmit": true,
-        "strictFunctionTypes": true
+        "strictFunctionTypes": true,
+        "resolveJsonModule": true
         // "strictNullChecks": true
     },
     "include": [