Use minecraft-data for protocol data (#126)

* use minecraft-data protocol data

* use minecraft-data for extra data

* Update .npmignore

* update skin data

* fix example

* remove .gitattr

* fix mcdata skin import, disable install script

.gitattributes

@@ -1 +0,0 @@
-data/*/*.json linguist-generated

.gitignore

@@ -4,10 +4,7 @@ package-lock.json
 __*
 src/**/*.json
 # Runtime generated data
-data/**/sample
-data/**/read.js
-data/**/write.js
-data/**/size.js
+data/
 tools/bds*
 tools/*/*
 *.txt

.npmignore

@@ -3,9 +3,9 @@ npm-debug.log
 __*
 src/**/*.json
 # Runtime generated data
-data/**/sample
+data/
 tools/bds*
 # Extra data
 examples
 test
-.github
+.github

data/provider.js

@@ -1,44 +0,0 @@
-const { Versions } = require('../src/options')
-const { getFiles } = require('../src/datatypes/util')
-const { join } = require('path')
-
-let fileMap = {}
-
-// Walks all the directories for each of the supported versions in options.js
-// then builds a file map for each version
-// { 'protocol.json': { '1.16.200': '1.16.200/protocol.json', '1.16.210': '1.16.210/...' } }
-function loadVersions () {
-  for (const version in Versions) {
-    let files = []
-    try {
-      files = getFiles(join(__dirname, '/', version))
-    } catch {}
-    for (const file of files) {
-      const rfile = file.replace(join(__dirname, '/', version) + '/', '')
-      fileMap[rfile] = fileMap[rfile] ?? []
-      fileMap[rfile].push([Versions[version], file])
-      fileMap[rfile].sort().reverse()
-    }
-  }
-}
-
-module.exports = (protocolVersion) => {
-  fileMap = {}
-  loadVersions()
-  return {
-    // Returns the most recent file based on the specified protocolVersion
-    // e.g. if `version` is 1.16 and a file for 1.16 doesn't exist, load from 1.15 file
-    getPath (file) {
-      if (!fileMap[file]) {
-        throw Error('Unknown file ' + file)
-      }
-      for (const [pver, path] of fileMap[file]) {
-        if (pver <= protocolVersion) {
-          // console.debug('for', file, 'returining', path)
-          return path
-        }
-      }
-      throw Error('unknown file ' + file)
-    }
-  }
-}

examples/server/server.js

@@ -16,9 +16,9 @@ process.env.DEBUG = 'minecraft-protocol' // packet logging
 const { Server } = require('bedrock-protocol')
 
 const { hasDumps } = require('../../tools/genPacketDumps')
-const DataProvider = require('../../data/provider')
 const { waitFor } = require('../../src/datatypes/util')
 const { loadWorld } = require('./serverChunks')
+const { join } = require('path')
 
 async function startServer (version = '1.17.10', ok) {
   if (!hasDumps(version)) {
@@ -30,7 +30,7 @@ async function startServer (version = '1.17.10', ok) {
   const server = new Server({ host: '0.0.0.0', port, version })
   let loop
 
-  const getPath = (packetPath) => DataProvider(server.options.protocolVersion).getPath(packetPath)
+  const getPath = (packetPath) => join(__dirname, `../data/${server.options.version}/${packetPath}`)
   const get = (packetName) => require(getPath(`sample/packets/${packetName}.json`))
 
   server.listen()

package.json

@@ -5,7 +5,6 @@
   "main": "index.js",
   "scripts": {
     "build": "cd tools && node compileProtocol.js",
-    "prepare": "npm run build",
     "test": "mocha --bail",
     "pretest": "npm run lint",
     "lint": "standard",
@@ -24,6 +23,7 @@
     "debug": "^4.3.1",
     "jsonwebtoken": "^8.5.1",
     "jsp-raknet": "^2.1.3",
+    "minecraft-data": "^2.89.4",
     "minecraft-folder-path": "^1.2.0",
     "prismarine-auth": "^1.1.0",
     "prismarine-nbt": "^1.5.0",

src/createClient.js

@@ -15,7 +15,7 @@ function createClient (options) {
     client.ping().then(data => {
       const advert = advertisement.fromServerName(data)
       console.log(`Connecting to server ${advert.motd} (${advert.name}), version ${advert.version}`)
-      client.version = options.version ?? advert.version
+      client.options.version = options.version ?? advert.version
       connect(client)
     }, client)
   }

src/handshake/login.js

@@ -1,15 +1,10 @@
-const fs = require('fs')
 const JWT = require('jsonwebtoken')
-const DataProvider = require('../../data/provider')
 const { nextUUID } = require('../datatypes/util')
 const { PUBLIC_KEY } = require('./constants')
 const algorithm = 'ES384'
 
 module.exports = (client, server, options) => {
-  const dp = DataProvider(options.protocolVersion)
-  const skinTex = fs.readFileSync(dp.getPath('steveSkin.bin')).toString('base64')
-  const skinGeom = fs.readFileSync(dp.getPath('steveGeometry.json')).toString('base64')
-  const skinData = JSON.parse(fs.readFileSync(dp.getPath('steve.json'), 'utf-8'))
+  const skinData = require('minecraft-data')('bedrock_' + options.version).defaultSkin
 
   client.createClientChain = (mojangKey, offline) => {
     const privateKey = client.ecdhKeyPair.privateKey
@@ -60,8 +55,6 @@ module.exports = (client, server, options) => {
 
       SelfSignedId: nextUUID(),
       ServerAddress: `${options.host}:${options.port}`,
-      SkinData: skinTex,
-      SkinGeometryData: skinGeom,
 
       ThirdPartyName: client.profile.name,
       ThirdPartyNameOnly: false,

src/options.js

@@ -1,15 +1,11 @@
+const mcData = require('minecraft-data')
+
 // Minimum supported version (< will be kicked)
 const MIN_VERSION = '1.16.201'
-// Currently supported verson
+// Currently supported verson. Note, clients with newer versions can still connect as long as data is in minecraft-data
 const CURRENT_VERSION = '1.17.10'
 
-const Versions = {
-  '1.17.10': 448,
-  '1.17.0': 440,
-  '1.16.220': 431,
-  '1.16.210': 428,
-  '1.16.201': 422
-}
+const Versions = Object.fromEntries(mcData.versions.bedrock.filter(e => e.releaseType === 'release').map(e => [e.minecraftVersion, e.version]))
 
 const defaultOptions = {
   // https://minecraft.gamepedia.com/Protocol_version#Bedrock_Edition_2

src/transforms/serializer.js

@@ -27,10 +27,13 @@ class Parser extends FullPacketParser {
 
 // Compiles the ProtoDef schema at runtime
 function createProtocol (version) {
-  const protocol = require(join(__dirname, `../../data/${version}/protocol.json`)).types
+  // Try and load from .js if available
+  try { require(`../../data/${version}/size.js`); return getProtocol(version) } catch {}
+
+  const protocol = require('minecraft-data')('bedrock_' + version).protocol
   const compiler = new ProtoDefCompiler()
-  compiler.addTypesToCompile(protocol)
-  compiler.addTypes(require(join(__dirname, '../datatypes/compiler-minecraft')))
+  compiler.addTypesToCompile(protocol.types)
+  compiler.addTypes(require('../datatypes/compiler-minecraft'))
   compiler.addTypes(require('prismarine-nbt/compiler-zigzag'))
 
   const compiledProto = compiler.compileProtoDefSync()
@@ -54,17 +57,18 @@ function getProtocol (version) {
 }
 
 function createSerializer (version) {
-  const proto = getProtocol(version)
+  const proto = createProtocol(version)
   return new Serializer(proto, 'mcpe_packet')
 }
 
 function createDeserializer (version) {
-  const proto = getProtocol(version)
+  const proto = createProtocol(version)
   return new Parser(proto, 'mcpe_packet')
 }
 
 module.exports = {
-  createDeserializer: createDeserializer,
-  createSerializer: createSerializer,
-  createProtocol: createProtocol
+  createDeserializer,
+  createSerializer,
+  createProtocol,
+  getProtocol
 }

test/internal.js

@@ -1,8 +1,8 @@
 const { Server, Client } = require('../')
 const { dumpPackets } = require('../tools/genPacketDumps')
-const DataProvider = require('../data/provider')
 const { ping } = require('../src/createClient')
 const { CURRENT_VERSION } = require('../src/options')
+const { join } = require('path')
 
 // First we need to dump some packets that a vanilla server would send a vanilla
 // client. Then we can replay those back in our custom server.
@@ -17,7 +17,7 @@ async function startTest (version = CURRENT_VERSION, ok) {
   const server = new Server({ host: '0.0.0.0', port, version, offline: true })
 
   function getPath (packetPath) {
-    return DataProvider(server.options.protocolVersion).getPath(packetPath)
+    return join(__dirname, `../data/${server.options.version}/${packetPath}`)
   }
 
   function get (packetPath) {

tools/compileProtocol.js

@@ -1,56 +1,19 @@
 /**
- * This is a utility script that converts the YAML here into ProtoDef schema code and (soon) docs/typescript definitions.
- * It also pre-compiles JS code from the schema for easier development.
- *
+ * Pre-compiles JS code from the schema for easier development.
  * You can run this with `npm run build`
- *
  */
 const fs = require('fs')
 const { ProtoDefCompiler } = require('protodef').Compiler
-const { Versions } = require('../src/options')
+const { convert } = require('minecraft-data/minecraft-data/tools/js/compileProtocol')
+const mcData = require('minecraft-data')
 const { join } = require('path')
-
-function getJSON (path) {
-  return JSON.parse(fs.readFileSync(path, 'utf-8'))
-}
-
-// Parse the YML files and turn to JSON
-function genProtoSchema () {
-  const { parse, compile } = require('protodef-yaml/compiler')
-
-  // Create the packet_map.yml from proto.yml
-  const parsed = parse('./proto.yml')
-  const version = parsed['!version']
-  const packets = []
-  for (const key in parsed) {
-    if (key.startsWith('%container')) {
-      const [, name] = key.split(',')
-      if (name.startsWith('packet_')) {
-        const children = parsed[key]
-        const packetName = name.replace('packet_', '')
-        const packetID = children['!id']
-        packets.push([packetID, packetName, name])
-      }
-    }
-  }
-  let l1 = ''
-  let l2 = ''
-  for (const [id, name, fname] of packets) {
-    l1 += `      0x${id.toString(16).padStart(2, '0')}: ${name}\n`
-    l2 += `      if ${name}: ${fname}\n`
-  }
-  // TODO: skip creating packet_map.yml and just generate the ProtoDef map JSON directly
-  const t = `#Auto-generated from proto.yml, do not modify\n!import: types.yaml\nmcpe_packet:\n   name: varint =>\n${l1}\n   params: name ?\n${l2}`
-  fs.writeFileSync('./packet_map.yml', t)
-
-  compile('./proto.yml', 'proto.json')
-  return version
-}
+// Filter versions we support
+const versions = mcData.versions.bedrock.filter(e => e.releaseType === 'release').map(e => e.minecraftVersion)
 
 // Compile the ProtoDef JSON into JS
-function createProtocol () {
+function createProtocol (version) {
   const compiler = new ProtoDefCompiler()
-  const protocol = getJSON('./protocol.json').types
+  const protocol = mcData('bedrock_' + version).protocol.types
   compiler.addTypes(require('../src/datatypes/compiler-minecraft'))
   compiler.addTypes(require('prismarine-nbt/compiler-zigzag'))
   compiler.addTypesToCompile(protocol)
@@ -63,27 +26,19 @@ function createProtocol () {
   return compiledProto
 }
 
-function copyLatest () {
-  process.chdir(join(__dirname, '/../data/latest'))
-  const version = genProtoSchema()
-  try { fs.mkdirSync(`../${version}`) } catch {}
-  fs.writeFileSync(`../${version}/protocol.json`, JSON.stringify({ types: getJSON('./proto.json') }, null, 2))
-  fs.unlinkSync('./proto.json') // remove temp file
-  fs.unlinkSync('./packet_map.yml') // remove temp file
-  return version
-}
-
 function main (ver = 'latest') {
-  if (ver === 'latest') ver = copyLatest()
-  process.chdir(join(__dirname, '/../data/', ver))
+  // Put the .js files into the data/ dir, we also use the data dir when dumping packets for tests
+  const dir = join(__dirname, '/../data/', ver)
+  if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true })
+  process.chdir(dir)
   console.log('Generating JS...', ver)
   createProtocol(ver)
 }
 
 // If no argument, build everything
 if (!process.argv[2]) {
-  copyLatest()
-  for (const version in Versions) {
+  convert('latest')
+  for (const version of versions) {
     main(version)
   }
 } else { // build the specified version