feat: implement basic sound system 🔊🔊🔊!

.github/workflows/preview.yml

@@ -29,6 +29,8 @@ jobs:
         run: vercel build --token=${{ secrets.VERCEL_TOKEN }}
       - name: Copy playground files
         run: node prismarine-viewer/esbuild.mjs && cp prismarine-viewer/public/index.html .vercel/output/static/playground.html && cp prismarine-viewer/public/playground.js .vercel/output/static/playground.js
+      - name: Download Generated Sounds map
+        run: node scripts/downloadSoundsMap.mjs
       - name: Deploy Project Artifacts to Vercel
         uses: mathiasvr/command-output@v2.0.0
         with:

.github/workflows/publish.yml

@@ -21,6 +21,8 @@ jobs:
       - run: vercel build --token=${{ secrets.VERCEL_TOKEN }} --prod
       - name: Copy playground files
         run: node prismarine-viewer/esbuild.mjs && cp prismarine-viewer/public/index.html .vercel/output/static/playground.html && cp prismarine-viewer/public/playground.js .vercel/output/static/playground.js
+      - name: Download Generated Sounds map
+        run: node scripts/downloadSoundsMap.mjs
       - name: Deploy Project to Vercel
         uses: mathiasvr/command-output@v2.0.0
         with:

prismarine-viewer/viewer/lib/viewer.ts

@@ -21,8 +21,9 @@ export class Viewer {
   isSneaking: boolean
   version: string
   cameraObjectOverride?: THREE.Object3D // for xr
+  audioListener: THREE.AudioListener
 
-  constructor (public renderer: THREE.WebGLRenderer, numWorkers?: number) {
+  constructor(public renderer: THREE.WebGLRenderer, numWorkers?: number) {
     this.scene = new THREE.Scene()
     this.scene.background = new THREE.Color('lightblue')
 
@@ -91,6 +92,33 @@ export class Viewer {
     cam.rotation.set(pitch, yaw, roll, 'ZYX')
   }
 
+  playSound (position: Vec3, path: string, volume = 1) {
+    if (!this.audioListener) {
+      this.audioListener = new THREE.AudioListener()
+      this.camera.add(this.audioListener)
+    }
+
+    const sound = new THREE.PositionalAudio(this.audioListener)
+
+    const audioLoader = new THREE.AudioLoader()
+    let start = Date.now()
+    audioLoader.loadAsync(path).then((buffer) => {
+      if (Date.now() - start > 500) return
+      // play
+      sound.setBuffer(buffer)
+      sound.setRefDistance(20)
+      sound.setVolume(volume)
+      this.scene.add(sound)
+      // set sound position
+      sound.position.set(position.x, position.y, position.z)
+      sound.play()
+      sound.onEnded = () => {
+        this.scene.remove(sound)
+        sound.disconnect()
+      }
+    })
+  }
+
   // todo type
   listen (emitter) {
     emitter.on('entity', (e) => {

scripts/downloadSoundsMap.mjs

@@ -0,0 +1,7 @@
+import fs from 'fs'
+
+const url = 'https://github.com/zardoy/prismarine-web-client/raw/sounds-generated/sounds.js'
+const savePath = 'dist/sounds.js'
+fetch(url).then(res => res.text()).then(data => {
+  fs.writeFileSync(savePath, data, 'utf8')
+})

scripts/prepareSounds.mjs

@@ -0,0 +1,243 @@
+//@ts-check
+
+import { getVersionList, DEFAULT_RESOURCE_ROOT_URL } from '@xmcl/installer'
+import path from 'path'
+import fs from 'fs'
+import { fileURLToPath } from 'url'
+import { exec } from 'child_process'
+import { promisify } from 'util'
+import { build } from 'esbuild'
+
+const __dirname = path.dirname(fileURLToPath(new URL(import.meta.url)))
+
+const targetedVersions = ['1.20.1', '1.19.2', '1.18.2', '1.17.1', '1.16.5', '1.15.2', '1.14.4', '1.13.2', '1.12.2', '1.11.2', '1.10.2', '1.9.4', '1.8.9']
+
+/** @type {{name, size, hash}[]} */
+let prevSounds = null
+
+const burgerDataUrl = (version) => `https://raw.githubusercontent.com/Pokechu22/Burger/gh-pages/${version}.json`
+const burgerDataPath = './generated/burger.json'
+
+// const perVersionData: Record<string, { removed: string[],
+
+const soundsPathVersionsRemap = {}
+
+const downloadAllSounds = async () => {
+  const { versions } = await getVersionList()
+  const lastVersion = versions.filter(version => !version.id.includes('w'))[0]
+  // if (lastVersion.id !== targetedVersions[0]) throw new Error('last version is not the same as targetedVersions[0], update')
+  for (const targetedVersion of targetedVersions) {
+    const versionData = versions.find(x => x.id === targetedVersion)
+    if (!versionData) throw new Error('no version data for ' + targetedVersion)
+    console.log('Getting assets for version', targetedVersion)
+    const { assetIndex } = await fetch(versionData.url).then((r) => r.json())
+    /** @type {{objects: {[a: string]: { size, hash }}}} */
+    const index = await fetch(assetIndex.url).then((r) => r.json())
+    const soundAssets = Object.entries(index.objects).filter(([name]) => /* name.endsWith('.ogg') || */ name.startsWith('minecraft/sounds/')).map(([name, { size, hash }]) => ({ name, size, hash }))
+    soundAssets.sort((a, b) => a.name.localeCompare(b.name))
+    if (prevSounds) {
+      const prevSoundNames = new Set(prevSounds.map(x => x.name))
+      const addedSounds = prevSounds.filter(x => !soundAssets.some(y => y.name === x.name))
+      // todo implement removed
+      const removedSounds = soundAssets.filter(x => !prevSoundNames.has(x.name))
+      // console.log('+', addedSounds.map(x => x.name))
+      // console.log('-', removedSounds.map(x => x.name))
+      const changedSize = soundAssets.filter(x => prevSoundNames.has(x.name) && prevSounds.find(y => y.name === x.name).size !== x.size)
+      console.log('changed size', changedSize.map(x => ({ name: x.name, prev: prevSounds.find(y => y.name === x.name).size, curr: x.size })))
+      if (addedSounds.length || changedSize.length) {
+        soundsPathVersionsRemap[targetedVersion] = [...addedSounds, ...changedSize].map(x => x.name.replace('minecraft/sounds/', '').replace('.ogg', ''))
+      }
+      if (addedSounds.length) {
+        console.log('downloading new sounds for version', targetedVersion)
+        downloadSounds(addedSounds, targetedVersion + '/')
+      }
+      if (changedSize.length) {
+        console.log('downloading changed sounds for version', targetedVersion)
+        downloadSounds(changedSize, targetedVersion + '/')
+      }
+    } else {
+      console.log('downloading sounds for version', targetedVersion)
+      downloadSounds(soundAssets)
+    }
+    prevSounds = soundAssets
+  }
+  async function downloadSound ({ name, hash, size }, namePath, log) {
+    const savePath = path.resolve(`generated/sounds/${namePath}`)
+    if (fs.existsSync(savePath)) {
+      // console.log('skipped', name)
+      return
+    }
+    log()
+    const r = await fetch(DEFAULT_RESOURCE_ROOT_URL + '/' + hash.slice(0, 2) + '/' + hash, /* {headers: {range: `bytes=0-${size-1}`}} */)
+    // save file
+    const file = await r.blob()
+    fs.mkdirSync(path.dirname(savePath), { recursive: true })
+    await fs.promises.writeFile(savePath, Buffer.from(await file.arrayBuffer()))
+
+    const reader = file.stream().getReader()
+
+    const writer = fs.createWriteStream(savePath)
+    let offset = 0
+    while (true) {
+      const { done, value } = await reader.read()
+      if (done) break
+      writer.write(Buffer.from(value))
+      offset += value.byteLength
+    }
+    writer.close()
+  }
+  async function downloadSounds (assets, addPath = '') {
+    for (let i = 0; i < assets.length; i += 5) {
+      await Promise.all(assets.slice(i, i + 5).map((asset, j) => downloadSound(asset, `${addPath}${asset.name}`, () => {
+        console.log('downloading', addPath, asset.name, i + j, '/', assets.length)
+      })))
+    }
+  }
+
+  fs.writeFileSync('./generated/soundsPathVersionsRemap.json', JSON.stringify(soundsPathVersionsRemap), 'utf8')
+}
+
+const lightpackOverrideSounds = {
+  'Block breaking': 'step/stone1',
+  'Block broken': 'dig/stone1',
+  'Block placed': 'dig/stone1'
+}
+
+// this is not done yet, will be used to select only sounds for bundle (most important ones)
+const isSoundWhitelisted = (name) => name.startsWith('random/') || name.startsWith('note/') || name.endsWith('/say1') || name.endsWith('/death') || (name.startsWith('mob/') && name.endsWith('/step1')) || name.endsWith('/swoop1') || /* name.endsWith('/break1') || */ name.endsWith('dig/stone1')
+
+const ffmpeg = 'C:/Users/Vitaly/Documents/LosslessCut-win-x64/resources/ffmpeg.exe' // will be ffmpeg-static
+const maintainBitrate = true
+
+const scanFilesDeep = async (root, onOggFile) => {
+  const files = await fs.promises.readdir(root, { withFileTypes: true })
+  for (const file of files) {
+    if (file.isDirectory()) {
+      await scanFilesDeep(path.join(root, file.name), onOggFile)
+    } else if (file.name.endsWith('.ogg') && !files.some(x => x.name === file.name.replace('.ogg', '.mp3'))) {
+      await onOggFile(path.join(root, file.name))
+    }
+  }
+}
+
+const convertSounds = async () => {
+  const toConvert = []
+  await scanFilesDeep('generated/sounds', (oggPath) => {
+    toConvert.push(oggPath)
+  })
+
+  const convertSound = async (i) => {
+    const proc = promisify(exec)(`${ffmpeg} -i "${toConvert[i]}" -y -codec:a libmp3lame ${maintainBitrate ? '-qscale:a 2' : ''} "${toConvert[i].replace('.ogg', '.mp3')}"`)
+    // pipe stdout to the console
+    proc.child.stdout.pipe(process.stdout)
+    await proc
+    console.log('converted to mp3', i, '/', toConvert.length, toConvert[i])
+  }
+
+  const CONCURRENCY = 5
+  for(let i = 0; i < toConvert.length; i += CONCURRENCY) {
+    await Promise.all(toConvert.slice(i, i + CONCURRENCY).map((oggPath, j) => convertSound(i + j)))
+  }
+}
+
+const getSoundsMap = (burgerData) => {
+  /** @type {Record<string, {id, name, sounds?: {name, weight?,volume?}[], subtitle?: string }>} */
+  return burgerData[0].sounds
+  // const map = JSON.parse(fs.readFileSync(burgerDataPath, 'utf8'))[0].sounds
+}
+
+const writeSoundsMap = async () => {
+  // const burgerData = await fetch(burgerDataUrl(targetedVersions[0])).then((r) => r.json())
+  // fs.writeFileSync(burgerDataPath, JSON.stringify(burgerData[0].sounds), 'utf8')
+
+  const allSoundsMapOutput = {}
+  let prevMap
+
+  // todo REMAP ONLY IDS. Do diffs, as mostly only ids are changed between versions
+  // const localTargetedVersions = targetedVersions.slice(0, 2)
+  const localTargetedVersions = targetedVersions
+  for (const targetedVersion of localTargetedVersions) {
+    const burgerData = await fetch(burgerDataUrl(targetedVersion)).then((r) => r.json())
+    const allSoundsMap = getSoundsMap(burgerData)
+    // console.log(Object.keys(sounds).length, 'ids')
+    const outputIdMap = {}
+    const outputFilesMap = {}
+
+    const classes = {}
+    let keysStats = {
+      new: 0,
+      same: 0
+    }
+    for (const { id, subtitle, sounds, name } of Object.values(allSoundsMap)) {
+      if (!sounds?.length /* && !subtitle */) continue
+      const firstName = sounds[0].name
+      // const includeSound = isSoundWhitelisted(firstName)
+      // if (!includeSound) continue
+      const mostUsedSound = sounds.sort((a, b) => b.weight - a.weight)[0]
+      const targetSound = sounds[0]
+      // outputMap[id] = { subtitle, sounds: mostUsedSound }
+      // outputMap[id] = { subtitle, sounds }
+      const soundFilePath = `generated/sounds/minecraft/sounds/${targetSound.name}.mp3`
+      // if (!fs.existsSync(soundFilePath)) {
+      //   console.warn('no sound file', targetSound.name)
+      //   continue
+      // }
+      const key = `${id};${name}`
+      outputIdMap[key] = `${targetSound.volume ?? 1};${targetSound.name}`
+      if (prevMap && prevMap[key]) {
+        keysStats.same++
+      } else {
+        keysStats.new++
+      }
+      // for (const {name: soundName} of sounds ?? []) {
+      //   let obj = classes
+      //   for (const part of soundName.split('/')) {
+      //     obj[part] ??= {}
+      //     obj = obj[part]
+      //   }
+      // }
+    }
+    // console.log(classes)
+    // console.log('to download', new Set(Object.values(outputIdMap).flatMap(x => x.sounds)).size)
+    // console.log('json size', JSON.stringify(outputIdMap).length / 1024 / 1024)
+    allSoundsMapOutput[targetedVersion] = outputIdMap
+    prevMap = outputIdMap
+    // const allSoundNames = new Set(Object.values(allSoundsMap).flatMap(({ name, sounds }) => {
+    //   if (!sounds) {
+    //     console.log(name)
+    //     return []
+    //   }
+    //   return sounds.map(sound => sound.name)
+    // }))
+    // console.log(allSoundNames.size, 'sounds')
+  }
+
+  fs.writeFileSync('./generated/sounds.json', JSON.stringify(allSoundsMapOutput), 'utf8')
+}
+
+const makeSoundsBundle = async () => {
+  const allSoundsMap = JSON.parse(fs.readFileSync('./generated/sounds.json', 'utf8'))
+  const allSoundsVersionedMap = JSON.parse(fs.readFileSync('./generated/soundsPathVersionsRemap.json', 'utf8'))
+
+  const allSoundsMeta = {
+    format: 'mp3',
+    baseUrl: 'https://raw.githubusercontent.com/zardoy/prismarine-web-client/sounds-generated/sounds/'
+  }
+
+  await build({
+    bundle: true,
+    outfile: `dist/sounds.js`,
+    stdin: {
+      contents: `window.allSoundsMap = ${JSON.stringify(allSoundsMap)}\nwindow.allSoundsVersionedMap = ${JSON.stringify(allSoundsVersionedMap)}\nwindow.allSoundsMeta = ${JSON.stringify(allSoundsMeta)}`,
+      resolveDir: __dirname,
+      sourcefile: `sounds.js`,
+      loader: 'js',
+    },
+    metafile: true,
+  })
+}
+
+// downloadAllSounds()
+// convertSounds()
+// writeSoundsMap()
+// makeSoundsBundle()

server.js

@@ -17,6 +17,8 @@ app.use(netApi({ allowOrigin: '*' }))
 if (!isProd) {
   app.use('/blocksStates', express.static(path.join(__dirname, './prismarine-viewer/public/blocksStates')))
   app.use('/textures', express.static(path.join(__dirname, './prismarine-viewer/public/textures')))
+
+  app.use('/sounds', express.static(path.join(__dirname, './generated/sounds/')))
 }
 // patch config
 app.get('/config.json', (req, res, next) => {

src/basicSounds.ts

@@ -7,7 +7,7 @@ const sounds: Record<string, any> = {}
 const loadingSounds = [] as string[]
 const convertedSounds = [] as string[]
 export async function loadSound (path: string) {
-  if (loadingSounds.includes(path)) return
+  if (loadingSounds.includes(path)) return true
   loadingSounds.push(path)
   const res = await window.fetch(path)
   const data = await res.arrayBuffer()
@@ -16,8 +16,19 @@ export async function loadSound (path: string) {
   loadingSounds.splice(loadingSounds.indexOf(path), 1)
 }
 
-export async function playSound (path) {
-  const volume = options.volume / 100
+export const loadOrPlaySound = async (url, soundVolume = 1) => {
+  const soundBuffer = sounds[url]
+  if (!soundBuffer) {
+    const start = Date.now()
+    const cancelled = await loadSound(url)
+    if (cancelled || Date.now() - start > 500) return
+  }
+
+  await playSound(url)
+}
+
+export async function playSound (url, soundVolume = 1) {
+  const volume = soundVolume * (options.volume / 100)
 
   if (!volume) return
 
@@ -29,9 +40,9 @@ export async function playSound (path) {
     convertedSounds.push(soundName)
   }
 
-  const soundBuffer = sounds[path]
+  const soundBuffer = sounds[url]
   if (!soundBuffer) {
-    console.warn(`Sound ${path} not loaded`)
+    console.warn(`Sound ${url} not loaded yet`)
     return
   }
 

src/globalState.ts

@@ -180,7 +180,7 @@ const initialNotification = {
 }
 export const notification = proxy(initialNotification)
 
-export const showNotification = (/** @type {Partial<typeof notification>} */newNotification) => {
+export const showNotification = (newNotification: Partial<typeof notification>) => {
   Object.assign(notification, { show: true, ...newNotification }, initialNotification)
 }
 

src/globals.d.ts

@@ -10,6 +10,10 @@ declare const localServer: import('flying-squid/dist/types').FullServer & { opti
 /** all currently loaded mc data */
 declare const mcData: Record<string, any>
 declare const loadedData: import('minecraft-data').IndexedData
+declare const customEvents: import('typed-emitter').default<{
+    singleplayer (): void
+    digStart()
+}>
 
 declare interface Document {
     getElementById (id): any

src/guessProblem.ts

@@ -1,4 +1,4 @@
-export const guessProblem = (/** @type {string} */errorMessage) => {
+export const guessProblem = (errorMessage: string) => {
   if (errorMessage.endsWith('Socket error: ECONNREFUSED')) {
     return 'Most probably the server is not running.'
   }

src/index.ts

@@ -82,8 +82,13 @@ import { fsState } from './loadSave'
 import { watchFov } from './rendererUtils'
 import { loadInMemorySave } from './react/SingleplayerProvider'
 
+// side effects
+import { downloadSoundsIfNeeded } from './soundSystem'
+import EventEmitter from 'events'
+
 window.debug = debug
 window.THREE = THREE
+window.customEvents = new EventEmitter()
 
 // ACTUAL CODE
 
@@ -342,6 +347,7 @@ async function connect (connectOptions: {
     Object.assign(serverOptions, connectOptions.serverOverridesFlat ?? {})
     const downloadMcData = async (version: string) => {
       setLoadingScreenStatus(`Downloading data for ${version}`)
+      await downloadSoundsIfNeeded()
       await loadScript(`./mc-data/${toMajorVersion(version)}.js`)
       miscUiState.loadedDataVersion = version
       try {

src/optionsGuiScheme.tsx

@@ -1,6 +1,6 @@
 import { useState } from 'react'
 import { useSnapshot } from 'valtio'
-import { miscUiState, openOptionsMenu } from './globalState'
+import { miscUiState, openOptionsMenu, showModal } from './globalState'
 import { openURL } from './menus/components/common'
 import { AppOptions, options } from './optionsStorage'
 import Button from './react/Button'
@@ -181,6 +181,11 @@ export const guiOptionsScheme: {
   ],
   sound: [
     { volume: {} },
+    {
+      custom () {
+        return <Button label='Sound Muffler' onClick={() => showModal({ reactType: 'sound-muffler' })} inScreen />
+      },
+    }
     // { ignoreSilentSwitch: {} },
   ],
   VR: [

src/optionsStorage.ts

@@ -48,7 +48,8 @@ const defaultOptions = {
   askGuestName: true,
 
   // advanced bot options
-  autoRespawn: false
+  autoRespawn: false,
+  mutedSounds: [] as string[]
 }
 
 export type AppOptions = typeof defaultOptions

src/react/AppStatusProvider.tsx

@@ -1,6 +1,6 @@
 import { proxy, useSnapshot } from 'valtio'
 import { useEffect } from 'react'
-import { activeModalStacks, hideModal, insertActiveModalStack, miscUiState } from '../globalState'
+import { activeModalStack, activeModalStacks, hideModal, insertActiveModalStack, miscUiState } from '../globalState'
 import { resetLocalStorageWorld } from '../browserfs'
 import { fsState } from '../loadSave'
 import AppStatus from './AppStatus'
@@ -43,14 +43,18 @@ export default () => {
   }, [isOpen])
 
   useEffect(() => {
+    const controller = new AbortController()
     window.addEventListener('keyup', (e) => {
-      if (!isOpen) return
+      if (activeModalStack.at(-1)?.reactType !== 'app-status') return
       if (e.code !== 'KeyR' || !lastConnectOptions.value) return
       window.dispatchEvent(new window.CustomEvent('connect', {
         detail: lastConnectOptions.value
       }))
       appStatusState.isError = false
-    }, {})
+    }, {
+      signal: controller.signal
+    })
+    return () => controller.abort()
   }, [])
 
   return <DiveTransition open={isOpen}>

src/react/ChatContainer.css

@@ -64,6 +64,7 @@ div.chat-wrapper { /* increase specificity */
 /* unsupported by firefox */
 ::-webkit-scrollbar {
     width: 5px;
+    height: 5px;
     background-color: rgb(24, 24, 24);
 }
 ::-webkit-scrollbar-thumb {

src/react/SoundMuffler.tsx

@@ -0,0 +1,60 @@
+import { useState } from 'react'
+import { useSnapshot } from 'valtio'
+import { hideCurrentModal } from '../globalState'
+import { lastPlayedSounds } from '../soundSystem'
+import { options } from '../optionsStorage'
+import Button from './Button'
+import Screen from './Screen'
+import { useIsModalActive } from './utils'
+
+const SoundRow = ({ sound, children }) => {
+  const { mutedSounds } = useSnapshot(options)
+
+  const isMuted = mutedSounds.includes(sound)
+
+  return <div style={{ display: 'flex', justifyContent: 'space-between', gap: 15 }}>
+    <div>
+      <span style={{ fontSize: 12, marginRight: 2, ...isMuted ? { color: '#af1c1c' } : {} }}>{sound}</span>
+      {children}
+    </div>
+    <Button icon={isMuted ? 'pixelarticons:music' : 'pixelarticons:close'} onClick={() => {
+      if (isMuted) {
+        options.mutedSounds.splice(options.mutedSounds.indexOf(sound), 1)
+      } else {
+        options.mutedSounds.push(sound)
+      }
+    }}></Button>
+  </div>
+}
+
+export default () => {
+  const isModalActive = useIsModalActive('sound-muffler')
+
+  const [showMuted, setShowMuted] = useState(true)
+  const [i, setI] = useState(0)
+  const { mutedSounds } = useSnapshot(options)
+
+  if (!isModalActive) return null
+
+  return <Screen title='Sound Muffler' backdrop>
+    <div style={{ display: 'flex', gap: 5, flexDirection: 'column' }}>
+      <Button onClick={() => setI(i => i + 1)}>Refresh</Button>
+      <Button onClick={() => setShowMuted(s => !s)}>Show Muted: {showMuted ? 'ON' : 'OFF'}</Button>
+      <span style={{ padding: '3px 0' }}>Last World Played</span>
+      {Object.entries(lastPlayedSounds.lastServerPlayed).map(([key, value]) => {
+        if (!showMuted && mutedSounds.includes(key)) return null as never
+        return [key, value.count] as const
+      }).filter(Boolean).sort((a, b) => b[1] - a[1]).slice(0, 20).map(([key, count]) => {
+        return <SoundRow key={key} sound={key}>
+          <span style={{ fontSize: 12, fontWeight: 'bold' }}>{count}</span>
+        </SoundRow>
+      })}
+      <span style={{ padding: '3px 0' }}>Last Client Played</span>
+      {lastPlayedSounds.lastClientPlayed.map((key) => {
+        if (!showMuted && mutedSounds.includes(key)) return null as never
+        return <SoundRow key={key} sound={key} children={undefined} />
+      })}
+      <Button onClick={() => hideCurrentModal()}>Back</Button>
+    </div>
+  </Screen>
+}

src/reactUi.jsx

@@ -18,6 +18,7 @@ import AppStatusProvider from './react/AppStatusProvider'
 import SelectOption from './react/SelectOption'
 import EnterFullscreenButton from './react/EnterFullscreenButton'
 import ChatProvider from './react/ChatProvider'
+import SoundMuffler from './react/SoundMuffler'
 
 // todo
 useInterfaceState.setState({
@@ -127,6 +128,7 @@ const InGameUi = () => {
       {/* apply scaling */}
       <DeathScreenProvider />
       <ChatProvider />
+      <SoundMuffler />
     </Portal>
     <DisplayQr />
     <Portal to={document.body}>

src/soundEvents.ts

@@ -1,36 +0,0 @@
-import type { Block } from 'prismarine-block'
-
-let lastStepSound = 0
-
-const getStepSound = (blockUnder: Block) => {
-  if (!blockUnder || blockUnder.type === 0) return
-  const soundsMap = window.allSoundsMap?.[bot.version]
-  if (!soundsMap) return
-  let soundResult = 'block.stone.step'
-  for (const x of Object.keys(soundsMap).map(n => n.split(';')[1])) {
-    const match = /block\.(.+)\.step/.exec(x)
-    const block = match?.[1]
-    if (!block) continue
-    if (loadedData.blocksByName[block]?.name === blockUnder.name) {
-      soundResult = x
-      break
-    }
-  }
-  return soundResult
-}
-
-export const movementHappening = () => {
-  const THRESHOLD = 0.1
-  const { x, z, y } = bot.player.entity.velocity
-  if (Math.abs(x) < THRESHOLD && (Math.abs(z) > THRESHOLD || Math.abs(y) > THRESHOLD)) {
-    // movement happening
-    if (Date.now() - lastStepSound > 500) {
-      const blockUnder = bot.world.getBlock(bot.entity.position.offset(0, -1, 0))
-      const stepSound = getStepSound(blockUnder)
-      if (stepSound) {
-        playHardcodedSound(stepSound)
-        lastStepSound = Date.now()
-      }
-    }
-  }
-}

src/soundSystem.ts

@@ -0,0 +1,229 @@
+import { subscribeKey } from 'valtio/utils'
+import { Vec3 } from 'vec3'
+import { versionToNumber } from 'prismarine-viewer/viewer/prepare/utils'
+import { loadScript } from 'prismarine-viewer/viewer/lib/utils'
+import type { Block } from 'prismarine-block'
+import { miscUiState, showNotification } from './globalState'
+import { options } from './optionsStorage'
+import { loadOrPlaySound } from './basicSounds'
+
+subscribeKey(miscUiState, 'gameLoaded', async () => {
+  const soundsLegacyMap = window.allSoundsVersionedMap as Record<string, string[]>
+  const allSoundsMap = window.allSoundsMap as Record<string, Record<string, string>>
+  const allSoundsMeta = window.allSoundsMeta as { format: string, baseUrl: string }
+  if (!allSoundsMap) {
+    return
+  }
+
+  // todo also use major versioned hardcoded sounds
+  const soundsMap = allSoundsMap[bot.version]
+
+  if (!soundsMap) {
+    console.warn('No sounds map for version', bot.version, 'supported versions are', Object.keys(allSoundsMap).join(', '))
+    showNotification({
+      message: 'No sounds map for version ' + bot.version,
+    })
+    return
+  }
+
+  if (!miscUiState.gameLoaded || !soundsMap) {
+    return
+  }
+
+  const soundsPerId = Object.fromEntries(Object.entries(soundsMap).map(([id, sound]) => [+id.split(';')[0], sound]))
+  const soundsPerName = Object.fromEntries(Object.entries(soundsMap).map(([id, sound]) => [id.split(';')[1], sound]))
+  const soundIdToName = Object.fromEntries(Object.entries(soundsMap).map(([id, sound]) => [+id.split(';')[0], id.split(';')[1]]))
+
+  const playGeneralSound = async (soundId: string, soundString: string | undefined, position?: Vec3, volume = 1, pitch?: number) => {
+    if (!soundString) {
+      console.warn('Unknown sound received from server to play', soundId)
+      return
+    }
+    if (!options.volume) return
+    console.debug('play sound', soundId)
+    const parts = soundString.split(';')
+    const soundVolume = +parts[0]!
+    const soundName = parts[1]!
+    // console.log('pitch', pitch)
+    const versionedSound = getVersionedSound(bot.version, soundName, Object.entries(soundsLegacyMap))
+    // todo test versionedSound
+    const url = allSoundsMeta.baseUrl.replace(/\/$/, '') + (versionedSound ? `/${versionedSound}` : '') + '/minecraft/sounds/' + soundName + '.' + allSoundsMeta.format
+    const soundKey = soundIdToName[+soundId] ?? soundId
+    const isMuted = options.mutedSounds.includes(soundKey)
+    if (position) {
+      if (!isMuted) {
+        viewer.playSound(position, url, soundVolume * volume * (options.volume / 100))
+      }
+      if (getDistance(bot.entity.position, position) < 16) {
+        lastPlayedSounds.lastServerPlayed[soundKey] ??= { count: 0, last: 0 }
+        lastPlayedSounds.lastServerPlayed[soundKey].count++
+        lastPlayedSounds.lastServerPlayed[soundKey].last = Date.now()
+      }
+    } else {
+      if (!isMuted) {
+        await loadOrPlaySound(url, volume)
+      }
+      lastPlayedSounds.lastClientPlayed.push(soundKey)
+      if (lastPlayedSounds.lastClientPlayed.length > 10) {
+        lastPlayedSounds.lastClientPlayed.shift()
+      }
+    }
+  }
+  const playHardcodedSound = async (soundId: string, position?: Vec3, volume = 1, pitch?: number) => {
+    const sound = soundsPerName[soundId]
+    await playGeneralSound(soundId, sound, position, volume, pitch)
+  }
+  bot.on('soundEffectHeard', async (soundId, position, volume, pitch) => {
+    console.debug('soundEffectHeard', soundId, volume)
+    await playHardcodedSound(soundId, position, volume, pitch)
+  })
+  bot.on('hardcodedSoundEffectHeard', async (soundId, soundCategory, position, volume, pitch) => {
+    const sound = soundsPerId[soundId]
+    await playGeneralSound(soundId.toString(), sound, position, volume, pitch)
+  })
+  bot.on('entityHurt', async (entity) => {
+    if (entity.id === bot.entity.id) {
+      await playHardcodedSound('entity.player.hurt')
+    }
+  })
+
+  const useBlockSound = (blockName: string, category: string, fallback: string) => {
+    blockName = {
+      // todo somehow generated, not full
+      grass_block: 'grass',
+      tall_grass: 'grass',
+      fern: 'grass',
+      large_fern: 'grass',
+      dead_bush: 'grass',
+      seagrass: 'grass',
+      tall_seagrass: 'grass',
+      kelp: 'grass',
+      kelp_plant: 'grass',
+      sugar_cane: 'grass',
+      bamboo: 'grass',
+      vine: 'grass',
+      nether_sprouts: 'grass',
+      nether_wart: 'grass',
+      twisting_vines: 'grass',
+      weeping_vines: 'grass',
+
+      cobblestone: 'stone',
+      stone_bricks: 'stone',
+      mossy_stone_bricks: 'stone',
+      cracked_stone_bricks: 'stone',
+      chiseled_stone_bricks: 'stone',
+      stone_brick_slab: 'stone',
+      stone_brick_stairs: 'stone',
+      stone_brick_wall: 'stone',
+      polished_granite: 'stone',
+    }[blockName] ?? blockName
+    const key = 'block.' + blockName + '.' + category
+    return soundsPerName[key] ? key : fallback
+  }
+
+  const getStepSound = (blockUnder: Block) => {
+    // const soundsMap = window.allSoundsMap?.[bot.version]
+    // if (!soundsMap) return
+    // let soundResult = 'block.stone.step'
+    // for (const x of Object.keys(soundsMap).map(n => n.split(';')[1])) {
+    //   const match = /block\.(.+)\.step/.exec(x)
+    //   const block = match?.[1]
+    //   if (!block) continue
+    //   if (loadedData.blocksByName[block]?.name === blockUnder.name) {
+    //     soundResult = x
+    //     break
+    //   }
+    // }
+    return useBlockSound(blockUnder.name, 'step', 'block.stone.step')
+  }
+
+  let lastStepSound = 0
+  const movementHappening = async () => {
+    const VELOCITY_THRESHOLD = 0.1
+    const { x, z, y } = bot.player.entity.velocity
+    if (bot.entity.onGround && Math.abs(x) < VELOCITY_THRESHOLD && (Math.abs(z) > VELOCITY_THRESHOLD || Math.abs(y) > VELOCITY_THRESHOLD)) {
+      // movement happening
+      if (Date.now() - lastStepSound > 300) {
+        const blockUnder = bot.world.getBlock(bot.entity.position.offset(0, -1, 0))
+        const stepSound = getStepSound(blockUnder)
+        if (stepSound) {
+          await playHardcodedSound(stepSound, undefined, 0.6)// todo not sure why 0.6
+          lastStepSound = Date.now()
+        }
+      }
+    }
+  }
+
+  const playBlockBreak = async (blockName: string, position?: Vec3) => {
+    const sound = useBlockSound(blockName, 'break', 'block.stone.break')
+
+    await playHardcodedSound(sound, position, 0.6, 1)
+  }
+
+  const registerEvents = () => {
+    bot.on('move', () => {
+      void movementHappening()
+    })
+    bot._client.on('world_event', async ({ effectId, location, data, global:disablePosVolume }) => {
+      const position = disablePosVolume ? undefined : new Vec3(location.x, location.y, location.z)
+      if (effectId === 2001) {
+        // break event
+        const block = loadedData.blocksByStateId[data]
+        await playBlockBreak(block.name, position)
+      }
+      // these produce glass break sound
+      if (effectId === 2002 || effectId === 2003 || effectId === 2007) {
+        await playHardcodedSound('block.glass.break', position, 1, 1)
+      }
+    })
+    let diggingBlock: Block | null = null
+    customEvents.on('digStart', () => {
+      diggingBlock = bot.blockAtCursor(5)
+    })
+    bot.on('diggingCompleted', async () => {
+      if (diggingBlock) {
+        await playBlockBreak(diggingBlock.name)
+      }
+    })
+  }
+
+  registerEvents()
+
+  console.log(getVersionedSound('1.13', 'mob/fox/spit1', Object.entries(soundsLegacyMap)))
+  // 1.20+ soundEffectHeard is broken atm
+  // bot._client.on('packet', (data, { name }, buffer) => {
+  //   if (name === 'sound_effect') {
+  //     console.log(data, buffer)
+  //   }
+  // })
+})
+
+const getVersionedSound = (version: string, item: string, itemsMapSortedEntries: Array<[string, string[]]>) => {
+  const verNumber = versionToNumber(version)
+  for (const [itemsVer, items] of itemsMapSortedEntries) {
+    // 1.18 < 1.18.1
+    // 1.13 < 1.13.2
+    if (items.includes(item) && verNumber <= versionToNumber(itemsVer)) {
+      return itemsVer
+    }
+  }
+}
+
+export const downloadSoundsIfNeeded = async () => {
+  if (!window.allSoundsMap) {
+    try {
+      await loadScript('./sounds.js')
+    } catch (err) {
+      console.warn('Sounds map was not generated. Sounds will not be played.')
+    }
+  }
+}
+
+export const lastPlayedSounds = {
+  lastClientPlayed: [] as string[],
+  lastServerPlayed: {} as Record<string, {count: number, last: number}>,
+}
+
+const getDistance = (pos1: Vec3, pos2: Vec3) => {
+  return Math.hypot((pos1.x - pos2.x), (pos1.y - pos2.y), (pos1.z - pos2.z))
+}

src/worldInteractions.js

@@ -195,6 +195,7 @@ class WorldInteraction {
         if (err.message === 'Digging aborted') return
         throw err
       })
+      customEvents.emit('digStart')
       this.lastDigged = Date.now()
     }
     this.prevOnGround = onGround