feat(big): items are now rendered in 3d not in 2d and it makes insanely huge difference on the game visuals

This commit is contained in:
Vitaly Turovsky 2025-08-15 05:26:11 +03:00
commit 54c114a702
4 changed files with 575 additions and 53 deletions

View file

@ -0,0 +1,13 @@
<!DOCTYPE html>
<html>
<head>
<title>Minecraft Item Viewer</title>
<style>
body { margin: 0; overflow: hidden; }
canvas { display: block; }
</style>
</head>
<body>
<script type="module" src="./three-item.ts"></script>
</body>
</html>

108
experiments/three-item.ts Normal file
View file

@ -0,0 +1,108 @@
import * as THREE from 'three'
import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls'
import itemsAtlas from 'mc-assets/dist/itemsAtlasLegacy.png'
import { createItemMeshFromCanvas, createItemMesh } from '../renderer/viewer/three/itemMesh'
// Create scene, camera and renderer
const scene = new THREE.Scene()
const camera = new THREE.PerspectiveCamera(75, window.innerWidth / window.innerHeight, 0.1, 1000)
const renderer = new THREE.WebGLRenderer({ antialias: true })
renderer.setSize(window.innerWidth, window.innerHeight)
document.body.appendChild(renderer.domElement)
// Setup camera and controls
camera.position.set(0, 0, 3)
const controls = new OrbitControls(camera, renderer.domElement)
controls.enableDamping = true
// Background and lights
scene.background = new THREE.Color(0x333333)
const ambientLight = new THREE.AmbientLight(0xffffff, 0.7)
scene.add(ambientLight)
// Animation loop
function animate () {
requestAnimationFrame(animate)
controls.update()
renderer.render(scene, camera)
}
async function setupItemMesh () {
try {
const loader = new THREE.TextureLoader()
const atlasTexture = await loader.loadAsync(itemsAtlas)
// Pixel-art configuration
atlasTexture.magFilter = THREE.NearestFilter
atlasTexture.minFilter = THREE.NearestFilter
atlasTexture.generateMipmaps = false
atlasTexture.wrapS = atlasTexture.wrapT = THREE.ClampToEdgeWrapping
// Extract the tile at x=2, y=0 (16x16)
const tileSize = 16
const tileX = 2
const tileY = 0
const canvas = document.createElement('canvas')
canvas.width = tileSize
canvas.height = tileSize
const ctx = canvas.getContext('2d')!
ctx.imageSmoothingEnabled = false
ctx.drawImage(
atlasTexture.image,
tileX * tileSize,
tileY * tileSize,
tileSize,
tileSize,
0,
0,
tileSize,
tileSize
)
// Test both approaches - working manual extraction:
const meshOld = createItemMeshFromCanvas(canvas, { depth: 0.1 })
meshOld.position.x = -1
meshOld.rotation.x = -Math.PI / 12
meshOld.rotation.y = Math.PI / 12
scene.add(meshOld)
// And new unified function:
const atlasWidth = atlasTexture.image.width
const atlasHeight = atlasTexture.image.height
const u = (tileX * tileSize) / atlasWidth
const v = (tileY * tileSize) / atlasHeight
const sizeX = tileSize / atlasWidth
const sizeY = tileSize / atlasHeight
console.log('Debug texture coords:', {u, v, sizeX, sizeY, atlasWidth, atlasHeight})
const resultNew = createItemMesh(atlasTexture, {
u, v, sizeX, sizeY
}, {
faceCamera: false,
use3D: true,
depth: 0.1
})
resultNew.mesh.position.x = 1
resultNew.mesh.rotation.x = -Math.PI / 12
resultNew.mesh.rotation.y = Math.PI / 12
scene.add(resultNew.mesh)
animate()
} catch (err) {
console.error('Failed to create item mesh:', err)
}
}
// Handle window resize
window.addEventListener('resize', () => {
camera.aspect = window.innerWidth / window.innerHeight
camera.updateProjectionMatrix()
renderer.setSize(window.innerWidth, window.innerHeight)
})
// Start
setupItemMesh()

View file

@ -21,6 +21,7 @@ import { loadSkinFromUsername, loadSkinImage, stevePngUrl } from '../lib/utils/s
import { renderComponent } from '../sign-renderer'
import { createCanvas } from '../lib/utils'
import { getBlockMeshFromModel } from './holdingBlock'
import { createItemMesh } from './itemMesh'
import * as Entity from './entity/EntityMesh'
import { getMesh } from './entity/EntityMesh'
import { WalkingGeneralSwing } from './entity/animations'
@ -741,71 +742,42 @@ export class Entities {
return {
mesh: outerGroup,
isBlock: true,
itemsTexture: null,
itemsTextureFlipped: null,
modelName: textureUv.modelName,
}
}
// TODO: Render proper model (especially for blocks) instead of flat texture
// Render proper 3D model for items
if (textureUv) {
const textureThree = textureUv.renderInfo?.texture === 'blocks' ? this.worldRenderer.material.map! : this.worldRenderer.itemsTexture
// todo use geometry buffer uv instead!
const { u, v, su, sv } = textureUv
const size = undefined
const itemsTexture = textureThree.clone()
itemsTexture.flipY = true
const sizeY = (sv ?? size)!
const sizeX = (su ?? size)!
itemsTexture.offset.set(u, 1 - v - sizeY)
itemsTexture.repeat.set(sizeX, sizeY)
itemsTexture.needsUpdate = true
itemsTexture.magFilter = THREE.NearestFilter
itemsTexture.minFilter = THREE.NearestFilter
let mesh: THREE.Object3D
let itemsTextureFlipped: THREE.Texture | undefined
if (faceCamera) {
const spriteMat = new THREE.SpriteMaterial({
map: itemsTexture,
transparent: true,
alphaTest: 0.1,
})
mesh = new THREE.Sprite(spriteMat)
} else {
itemsTextureFlipped = itemsTexture.clone()
itemsTextureFlipped.repeat.x *= -1
itemsTextureFlipped.needsUpdate = true
itemsTextureFlipped.offset.set(u + (sizeX), 1 - v - sizeY)
const material = new THREE.MeshStandardMaterial({
map: itemsTexture,
transparent: true,
alphaTest: 0.1,
})
const materialFlipped = new THREE.MeshStandardMaterial({
map: itemsTextureFlipped,
transparent: true,
alphaTest: 0.1,
})
mesh = new THREE.Mesh(new THREE.BoxGeometry(1, 1, 0), [
// top left and right bottom are black box materials others are transparent
new THREE.MeshBasicMaterial({ color: 0x00_00_00 }), new THREE.MeshBasicMaterial({ color: 0x00_00_00 }),
new THREE.MeshBasicMaterial({ color: 0x00_00_00 }), new THREE.MeshBasicMaterial({ color: 0x00_00_00 }),
material, materialFlipped,
])
}
const sizeX = su ?? 1 // su is actually width
const sizeY = sv ?? 1 // sv is actually height
// Use the new unified item mesh function
const result = createItemMesh(textureThree, {
u,
v,
sizeX,
sizeY
}, {
faceCamera,
use3D: !faceCamera, // Only use 3D for non-camera-facing items
depth: 0.03
})
let SCALE = 1
if (specificProps['minecraft:display_context'] === 'ground') {
SCALE = 0.5
} else if (specificProps['minecraft:display_context'] === 'thirdperson') {
SCALE = 6
}
mesh.scale.set(SCALE, SCALE, SCALE)
result.mesh.scale.set(SCALE, SCALE, SCALE)
return {
mesh,
mesh: result.mesh,
isBlock: false,
itemsTexture,
itemsTextureFlipped,
modelName: textureUv.modelName,
cleanup: result.cleanup
}
}
}
@ -905,8 +877,9 @@ export class Entities {
group.additionalCleanup = () => {
// important: avoid texture memory leak and gpu slowdown
object.itemsTexture?.dispose()
object.itemsTextureFlipped?.dispose()
if (object.cleanup) {
object.cleanup()
}
}
}
}
@ -1295,8 +1268,9 @@ export class Entities {
const group = new THREE.Object3D()
group['additionalCleanup'] = () => {
// important: avoid texture memory leak and gpu slowdown
itemObject.itemsTexture?.dispose()
itemObject.itemsTextureFlipped?.dispose()
if (itemObject.cleanup) {
itemObject.cleanup()
}
}
const itemMesh = itemObject.mesh
group.rotation.z = -Math.PI / 16

View file

@ -0,0 +1,427 @@
import * as THREE from 'three'
export interface Create3DItemMeshOptions {
depth?: number
pixelSize?: number
}
export interface Create3DItemMeshResult {
geometry: THREE.BufferGeometry
totalVertices: number
totalTriangles: number
}
/**
* Creates a 3D item geometry with front/back faces and connecting edges
* from a canvas containing the item texture
*/
export function create3DItemMesh (
canvas: HTMLCanvasElement,
options: Create3DItemMeshOptions = {}
): Create3DItemMeshResult {
const { depth = 0.03, pixelSize } = options
// Validate canvas dimensions
if (canvas.width <= 0 || canvas.height <= 0) {
throw new Error(`Invalid canvas dimensions: ${canvas.width}x${canvas.height}`)
}
const ctx = canvas.getContext('2d')!
const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height)
const { data } = imageData
const w = canvas.width
const h = canvas.height
const halfDepth = depth / 2
const actualPixelSize = pixelSize ?? (1 / Math.max(w, h))
// Find opaque pixels
const isOpaque = (x: number, y: number) => {
if (x < 0 || y < 0 || x >= w || y >= h) return false
const i = (y * w + x) * 4
return data[i + 3] > 128 // alpha > 128
}
const vertices: number[] = []
const indices: number[] = []
const uvs: number[] = []
const normals: number[] = []
let vertexIndex = 0
// Helper to add a vertex
const addVertex = (x: number, y: number, z: number, u: number, v: number, nx: number, ny: number, nz: number) => {
vertices.push(x, y, z)
uvs.push(u, v)
normals.push(nx, ny, nz)
return vertexIndex++
}
// Helper to add a quad (two triangles)
const addQuad = (v0: number, v1: number, v2: number, v3: number) => {
indices.push(v0, v1, v2, v0, v2, v3)
}
// Convert pixel coordinates to world coordinates
const pixelToWorld = (px: number, py: number) => {
const x = (px / w - 0.5) * actualPixelSize * w
const y = -(py / h - 0.5) * actualPixelSize * h
return { x, y }
}
// Create a grid of vertices for front and back faces
const frontVertices: Array<Array<number | null>> = Array.from({ length: h + 1 }, () => Array.from({ length: w + 1 }, () => null))
const backVertices: Array<Array<number | null>> = Array.from({ length: h + 1 }, () => Array.from({ length: w + 1 }, () => null))
// Create vertices at pixel corners
for (let py = 0; py <= h; py++) {
for (let px = 0; px <= w; px++) {
const { x, y } = pixelToWorld(px - 0.5, py - 0.5)
// UV coordinates should map to the texture space of the extracted tile
const u = px / w
const v = py / h
// Check if this vertex is needed for any face or edge
let needVertex = false
// Check all 4 adjacent pixels to see if any are opaque
const adjacentPixels = [
[px - 1, py - 1], // top-left pixel
[px, py - 1], // top-right pixel
[px - 1, py], // bottom-left pixel
[px, py] // bottom-right pixel
]
for (const [adjX, adjY] of adjacentPixels) {
if (isOpaque(adjX, adjY)) {
needVertex = true
break
}
}
if (needVertex) {
frontVertices[py][px] = addVertex(x, y, halfDepth, u, v, 0, 0, 1)
backVertices[py][px] = addVertex(x, y, -halfDepth, u, v, 0, 0, -1)
}
}
}
// Create front and back faces
for (let py = 0; py < h; py++) {
for (let px = 0; px < w; px++) {
if (!isOpaque(px, py)) continue
const v00 = frontVertices[py][px]
const v10 = frontVertices[py][px + 1]
const v11 = frontVertices[py + 1][px + 1]
const v01 = frontVertices[py + 1][px]
const b00 = backVertices[py][px]
const b10 = backVertices[py][px + 1]
const b11 = backVertices[py + 1][px + 1]
const b01 = backVertices[py + 1][px]
if (v00 !== null && v10 !== null && v11 !== null && v01 !== null) {
// Front face
addQuad(v00, v10, v11, v01)
}
if (b00 !== null && b10 !== null && b11 !== null && b01 !== null) {
// Back face (reversed winding)
addQuad(b10, b00, b01, b11)
}
}
}
// Create edge faces for each side of the pixel with proper UVs
for (let py = 0; py < h; py++) {
for (let px = 0; px < w; px++) {
if (!isOpaque(px, py)) continue
const pixelU = (px + 0.5) / w // Center of current pixel
const pixelV = (py + 0.5) / h
// Left edge (x = px)
if (!isOpaque(px - 1, py)) {
const f0 = frontVertices[py][px]
const f1 = frontVertices[py + 1][px]
const b0 = backVertices[py][px]
const b1 = backVertices[py + 1][px]
if (f0 !== null && f1 !== null && b0 !== null && b1 !== null) {
// Create new vertices for edge with current pixel's UV
const ef0 = addVertex(vertices[f0 * 3], vertices[f0 * 3 + 1], vertices[f0 * 3 + 2], pixelU, pixelV, -1, 0, 0)
const ef1 = addVertex(vertices[f1 * 3], vertices[f1 * 3 + 1], vertices[f1 * 3 + 2], pixelU, pixelV, -1, 0, 0)
const eb1 = addVertex(vertices[b1 * 3], vertices[b1 * 3 + 1], vertices[b1 * 3 + 2], pixelU, pixelV, -1, 0, 0)
const eb0 = addVertex(vertices[b0 * 3], vertices[b0 * 3 + 1], vertices[b0 * 3 + 2], pixelU, pixelV, -1, 0, 0)
addQuad(ef0, ef1, eb1, eb0)
}
}
// Right edge (x = px + 1)
if (!isOpaque(px + 1, py)) {
const f0 = frontVertices[py + 1][px + 1]
const f1 = frontVertices[py][px + 1]
const b0 = backVertices[py + 1][px + 1]
const b1 = backVertices[py][px + 1]
if (f0 !== null && f1 !== null && b0 !== null && b1 !== null) {
const ef0 = addVertex(vertices[f0 * 3], vertices[f0 * 3 + 1], vertices[f0 * 3 + 2], pixelU, pixelV, 1, 0, 0)
const ef1 = addVertex(vertices[f1 * 3], vertices[f1 * 3 + 1], vertices[f1 * 3 + 2], pixelU, pixelV, 1, 0, 0)
const eb1 = addVertex(vertices[b1 * 3], vertices[b1 * 3 + 1], vertices[b1 * 3 + 2], pixelU, pixelV, 1, 0, 0)
const eb0 = addVertex(vertices[b0 * 3], vertices[b0 * 3 + 1], vertices[b0 * 3 + 2], pixelU, pixelV, 1, 0, 0)
addQuad(ef0, ef1, eb1, eb0)
}
}
// Top edge (y = py)
if (!isOpaque(px, py - 1)) {
const f0 = frontVertices[py][px]
const f1 = frontVertices[py][px + 1]
const b0 = backVertices[py][px]
const b1 = backVertices[py][px + 1]
if (f0 !== null && f1 !== null && b0 !== null && b1 !== null) {
const ef0 = addVertex(vertices[f0 * 3], vertices[f0 * 3 + 1], vertices[f0 * 3 + 2], pixelU, pixelV, 0, -1, 0)
const ef1 = addVertex(vertices[f1 * 3], vertices[f1 * 3 + 1], vertices[f1 * 3 + 2], pixelU, pixelV, 0, -1, 0)
const eb1 = addVertex(vertices[b1 * 3], vertices[b1 * 3 + 1], vertices[b1 * 3 + 2], pixelU, pixelV, 0, -1, 0)
const eb0 = addVertex(vertices[b0 * 3], vertices[b0 * 3 + 1], vertices[b0 * 3 + 2], pixelU, pixelV, 0, -1, 0)
addQuad(ef0, ef1, eb1, eb0)
}
}
// Bottom edge (y = py + 1)
if (!isOpaque(px, py + 1)) {
const f0 = frontVertices[py + 1][px + 1]
const f1 = frontVertices[py + 1][px]
const b0 = backVertices[py + 1][px + 1]
const b1 = backVertices[py + 1][px]
if (f0 !== null && f1 !== null && b0 !== null && b1 !== null) {
const ef0 = addVertex(vertices[f0 * 3], vertices[f0 * 3 + 1], vertices[f0 * 3 + 2], pixelU, pixelV, 0, 1, 0)
const ef1 = addVertex(vertices[f1 * 3], vertices[f1 * 3 + 1], vertices[f1 * 3 + 2], pixelU, pixelV, 0, 1, 0)
const eb1 = addVertex(vertices[b1 * 3], vertices[b1 * 3 + 1], vertices[b1 * 3 + 2], pixelU, pixelV, 0, 1, 0)
const eb0 = addVertex(vertices[b0 * 3], vertices[b0 * 3 + 1], vertices[b0 * 3 + 2], pixelU, pixelV, 0, 1, 0)
addQuad(ef0, ef1, eb1, eb0)
}
}
}
}
const geometry = new THREE.BufferGeometry()
geometry.setAttribute('position', new THREE.Float32BufferAttribute(vertices, 3))
geometry.setAttribute('uv', new THREE.Float32BufferAttribute(uvs, 2))
geometry.setAttribute('normal', new THREE.Float32BufferAttribute(normals, 3))
geometry.setIndex(indices)
// Compute normals properly
geometry.computeVertexNormals()
return {
geometry,
totalVertices: vertexIndex,
totalTriangles: indices.length / 3
}
}
export interface ItemTextureInfo {
u: number
v: number
sizeX: number
sizeY: number
}
export interface ItemMeshResult {
mesh: THREE.Object3D
itemsTexture?: THREE.Texture
itemsTextureFlipped?: THREE.Texture
cleanup?: () => void
}
/**
* Extracts item texture region to a canvas
*/
export function extractItemTextureToCanvas (
sourceTexture: THREE.Texture,
textureInfo: ItemTextureInfo
): HTMLCanvasElement {
const { u, v, sizeX, sizeY } = textureInfo
// Calculate canvas size - fix the calculation
const canvasWidth = Math.max(1, Math.floor(sizeX * sourceTexture.image.width))
const canvasHeight = Math.max(1, Math.floor(sizeY * sourceTexture.image.height))
const canvas = document.createElement('canvas')
canvas.width = canvasWidth
canvas.height = canvasHeight
const ctx = canvas.getContext('2d')!
ctx.imageSmoothingEnabled = false
// Draw the item texture region to canvas
ctx.drawImage(
sourceTexture.image,
u * sourceTexture.image.width,
v * sourceTexture.image.height,
sizeX * sourceTexture.image.width,
sizeY * sourceTexture.image.height,
0,
0,
canvas.width,
canvas.height
)
return canvas
}
/**
* Creates either a 2D or 3D item mesh based on parameters
*/
export function createItemMesh (
sourceTexture: THREE.Texture,
textureInfo: ItemTextureInfo,
options: {
faceCamera?: boolean
use3D?: boolean
depth?: number
} = {}
): ItemMeshResult {
const { faceCamera = false, use3D = true, depth = 0.03 } = options
const { u, v, sizeX, sizeY } = textureInfo
if (faceCamera) {
// Create sprite for camera-facing items
const itemsTexture = sourceTexture.clone()
itemsTexture.flipY = true
itemsTexture.offset.set(u, 1 - v - sizeY)
itemsTexture.repeat.set(sizeX, sizeY)
itemsTexture.needsUpdate = true
itemsTexture.magFilter = THREE.NearestFilter
itemsTexture.minFilter = THREE.NearestFilter
const spriteMat = new THREE.SpriteMaterial({
map: itemsTexture,
transparent: true,
alphaTest: 0.1,
})
const mesh = new THREE.Sprite(spriteMat)
return {
mesh,
itemsTexture,
cleanup () {
itemsTexture.dispose()
}
}
}
if (use3D) {
// Try to create 3D mesh
try {
const canvas = extractItemTextureToCanvas(sourceTexture, textureInfo)
const { geometry } = create3DItemMesh(canvas, { depth })
// Create texture from canvas for the 3D mesh
const itemsTexture = new THREE.CanvasTexture(canvas)
itemsTexture.magFilter = THREE.NearestFilter
itemsTexture.minFilter = THREE.NearestFilter
itemsTexture.wrapS = itemsTexture.wrapT = THREE.ClampToEdgeWrapping
itemsTexture.flipY = false
itemsTexture.needsUpdate = true
const material = new THREE.MeshStandardMaterial({
map: itemsTexture,
side: THREE.DoubleSide,
transparent: true,
alphaTest: 0.1,
})
const mesh = new THREE.Mesh(geometry, material)
return {
mesh,
itemsTexture,
cleanup () {
itemsTexture.dispose()
geometry.dispose()
if (material.map) material.map.dispose()
material.dispose()
}
}
} catch (error) {
console.warn('Failed to create 3D item mesh, falling back to 2D:', error)
// Fall through to 2D rendering
}
}
// Fallback to 2D flat rendering
const itemsTexture = sourceTexture.clone()
itemsTexture.flipY = true
itemsTexture.offset.set(u, 1 - v - sizeY)
itemsTexture.repeat.set(sizeX, sizeY)
itemsTexture.needsUpdate = true
itemsTexture.magFilter = THREE.NearestFilter
itemsTexture.minFilter = THREE.NearestFilter
const itemsTextureFlipped = itemsTexture.clone()
itemsTextureFlipped.repeat.x *= -1
itemsTextureFlipped.needsUpdate = true
itemsTextureFlipped.offset.set(u + sizeX, 1 - v - sizeY)
const material = new THREE.MeshStandardMaterial({
map: itemsTexture,
transparent: true,
alphaTest: 0.1,
})
const materialFlipped = new THREE.MeshStandardMaterial({
map: itemsTextureFlipped,
transparent: true,
alphaTest: 0.1,
})
const mesh = new THREE.Mesh(new THREE.BoxGeometry(1, 1, 0), [
new THREE.MeshBasicMaterial({ color: 0x00_00_00 }), new THREE.MeshBasicMaterial({ color: 0x00_00_00 }),
new THREE.MeshBasicMaterial({ color: 0x00_00_00 }), new THREE.MeshBasicMaterial({ color: 0x00_00_00 }),
material, materialFlipped,
])
return {
mesh,
itemsTexture,
itemsTextureFlipped,
cleanup () {
itemsTexture.dispose()
itemsTextureFlipped.dispose()
material.dispose()
materialFlipped.dispose()
}
}
}
/**
* Creates a complete 3D item mesh from a canvas texture
*/
export function createItemMeshFromCanvas (
canvas: HTMLCanvasElement,
options: Create3DItemMeshOptions = {}
): THREE.Mesh {
const { geometry } = create3DItemMesh(canvas, options)
// Base color texture for the item
const colorTexture = new THREE.CanvasTexture(canvas)
colorTexture.magFilter = THREE.NearestFilter
colorTexture.minFilter = THREE.NearestFilter
colorTexture.wrapS = colorTexture.wrapT = THREE.ClampToEdgeWrapping
colorTexture.flipY = false // Important for canvas textures
colorTexture.needsUpdate = true
// Material - no transparency, no alpha test needed for edges
const material = new THREE.MeshBasicMaterial({
map: colorTexture,
side: THREE.DoubleSide,
transparent: true,
alphaTest: 0.1
})
return new THREE.Mesh(geometry, material)
}