diff --git a/experiments/three-item.html b/experiments/three-item.html new file mode 100644 index 00000000..70155c50 --- /dev/null +++ b/experiments/three-item.html @@ -0,0 +1,13 @@ + + + + Minecraft Item Viewer + + + + + + diff --git a/experiments/three-item.ts b/experiments/three-item.ts new file mode 100644 index 00000000..b9d492fe --- /dev/null +++ b/experiments/three-item.ts @@ -0,0 +1,108 @@ +import * as THREE from 'three' +import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls' +import itemsAtlas from 'mc-assets/dist/itemsAtlasLegacy.png' +import { createItemMeshFromCanvas, createItemMesh } from '../renderer/viewer/three/itemMesh' + +// Create scene, camera and renderer +const scene = new THREE.Scene() +const camera = new THREE.PerspectiveCamera(75, window.innerWidth / window.innerHeight, 0.1, 1000) +const renderer = new THREE.WebGLRenderer({ antialias: true }) +renderer.setSize(window.innerWidth, window.innerHeight) +document.body.appendChild(renderer.domElement) + +// Setup camera and controls +camera.position.set(0, 0, 3) +const controls = new OrbitControls(camera, renderer.domElement) +controls.enableDamping = true + +// Background and lights +scene.background = new THREE.Color(0x333333) +const ambientLight = new THREE.AmbientLight(0xffffff, 0.7) +scene.add(ambientLight) + +// Animation loop +function animate () { + requestAnimationFrame(animate) + controls.update() + renderer.render(scene, camera) +} + +async function setupItemMesh () { + try { + const loader = new THREE.TextureLoader() + const atlasTexture = await loader.loadAsync(itemsAtlas) + + // Pixel-art configuration + atlasTexture.magFilter = THREE.NearestFilter + atlasTexture.minFilter = THREE.NearestFilter + atlasTexture.generateMipmaps = false + atlasTexture.wrapS = atlasTexture.wrapT = THREE.ClampToEdgeWrapping + + // Extract the tile at x=2, y=0 (16x16) + const tileSize = 16 + const tileX = 2 + const tileY = 0 + + const canvas = document.createElement('canvas') + canvas.width = tileSize + canvas.height = tileSize + const ctx = canvas.getContext('2d')! + + ctx.imageSmoothingEnabled = false + ctx.drawImage( + atlasTexture.image, + tileX * tileSize, + tileY * tileSize, + tileSize, + tileSize, + 0, + 0, + tileSize, + tileSize + ) + + // Test both approaches - working manual extraction: + const meshOld = createItemMeshFromCanvas(canvas, { depth: 0.1 }) + meshOld.position.x = -1 + meshOld.rotation.x = -Math.PI / 12 + meshOld.rotation.y = Math.PI / 12 + scene.add(meshOld) + + // And new unified function: + const atlasWidth = atlasTexture.image.width + const atlasHeight = atlasTexture.image.height + const u = (tileX * tileSize) / atlasWidth + const v = (tileY * tileSize) / atlasHeight + const sizeX = tileSize / atlasWidth + const sizeY = tileSize / atlasHeight + + console.log('Debug texture coords:', {u, v, sizeX, sizeY, atlasWidth, atlasHeight}) + + const resultNew = createItemMesh(atlasTexture, { + u, v, sizeX, sizeY + }, { + faceCamera: false, + use3D: true, + depth: 0.1 + }) + + resultNew.mesh.position.x = 1 + resultNew.mesh.rotation.x = -Math.PI / 12 + resultNew.mesh.rotation.y = Math.PI / 12 + scene.add(resultNew.mesh) + + animate() + } catch (err) { + console.error('Failed to create item mesh:', err) + } +} + +// Handle window resize +window.addEventListener('resize', () => { + camera.aspect = window.innerWidth / window.innerHeight + camera.updateProjectionMatrix() + renderer.setSize(window.innerWidth, window.innerHeight) +}) + +// Start +setupItemMesh() diff --git a/renderer/viewer/three/entities.ts b/renderer/viewer/three/entities.ts index 24f64803..b1828d92 100644 --- a/renderer/viewer/three/entities.ts +++ b/renderer/viewer/three/entities.ts @@ -21,6 +21,7 @@ import { loadSkinFromUsername, loadSkinImage, stevePngUrl } from '../lib/utils/s import { renderComponent } from '../sign-renderer' import { createCanvas } from '../lib/utils' import { getBlockMeshFromModel } from './holdingBlock' +import { createItemMesh } from './itemMesh' import * as Entity from './entity/EntityMesh' import { getMesh } from './entity/EntityMesh' import { WalkingGeneralSwing } from './entity/animations' @@ -741,71 +742,42 @@ export class Entities { return { mesh: outerGroup, isBlock: true, - itemsTexture: null, - itemsTextureFlipped: null, modelName: textureUv.modelName, } } - // TODO: Render proper model (especially for blocks) instead of flat texture + // Render proper 3D model for items if (textureUv) { const textureThree = textureUv.renderInfo?.texture === 'blocks' ? this.worldRenderer.material.map! : this.worldRenderer.itemsTexture - // todo use geometry buffer uv instead! const { u, v, su, sv } = textureUv - const size = undefined - const itemsTexture = textureThree.clone() - itemsTexture.flipY = true - const sizeY = (sv ?? size)! - const sizeX = (su ?? size)! - itemsTexture.offset.set(u, 1 - v - sizeY) - itemsTexture.repeat.set(sizeX, sizeY) - itemsTexture.needsUpdate = true - itemsTexture.magFilter = THREE.NearestFilter - itemsTexture.minFilter = THREE.NearestFilter - let mesh: THREE.Object3D - let itemsTextureFlipped: THREE.Texture | undefined - if (faceCamera) { - const spriteMat = new THREE.SpriteMaterial({ - map: itemsTexture, - transparent: true, - alphaTest: 0.1, - }) - mesh = new THREE.Sprite(spriteMat) - } else { - itemsTextureFlipped = itemsTexture.clone() - itemsTextureFlipped.repeat.x *= -1 - itemsTextureFlipped.needsUpdate = true - itemsTextureFlipped.offset.set(u + (sizeX), 1 - v - sizeY) - const material = new THREE.MeshStandardMaterial({ - map: itemsTexture, - transparent: true, - alphaTest: 0.1, - }) - const materialFlipped = new THREE.MeshStandardMaterial({ - map: itemsTextureFlipped, - transparent: true, - alphaTest: 0.1, - }) - mesh = new THREE.Mesh(new THREE.BoxGeometry(1, 1, 0), [ - // top left and right bottom are black box materials others are transparent - new THREE.MeshBasicMaterial({ color: 0x00_00_00 }), new THREE.MeshBasicMaterial({ color: 0x00_00_00 }), - new THREE.MeshBasicMaterial({ color: 0x00_00_00 }), new THREE.MeshBasicMaterial({ color: 0x00_00_00 }), - material, materialFlipped, - ]) - } + const sizeX = su ?? 1 // su is actually width + const sizeY = sv ?? 1 // sv is actually height + + // Use the new unified item mesh function + const result = createItemMesh(textureThree, { + u, + v, + sizeX, + sizeY + }, { + faceCamera, + use3D: !faceCamera, // Only use 3D for non-camera-facing items + depth: 0.03 + }) + let SCALE = 1 if (specificProps['minecraft:display_context'] === 'ground') { SCALE = 0.5 } else if (specificProps['minecraft:display_context'] === 'thirdperson') { SCALE = 6 } - mesh.scale.set(SCALE, SCALE, SCALE) + result.mesh.scale.set(SCALE, SCALE, SCALE) + return { - mesh, + mesh: result.mesh, isBlock: false, - itemsTexture, - itemsTextureFlipped, modelName: textureUv.modelName, + cleanup: result.cleanup } } } @@ -905,8 +877,9 @@ export class Entities { group.additionalCleanup = () => { // important: avoid texture memory leak and gpu slowdown - object.itemsTexture?.dispose() - object.itemsTextureFlipped?.dispose() + if (object.cleanup) { + object.cleanup() + } } } } @@ -1295,8 +1268,9 @@ export class Entities { const group = new THREE.Object3D() group['additionalCleanup'] = () => { // important: avoid texture memory leak and gpu slowdown - itemObject.itemsTexture?.dispose() - itemObject.itemsTextureFlipped?.dispose() + if (itemObject.cleanup) { + itemObject.cleanup() + } } const itemMesh = itemObject.mesh group.rotation.z = -Math.PI / 16 diff --git a/renderer/viewer/three/itemMesh.ts b/renderer/viewer/three/itemMesh.ts new file mode 100644 index 00000000..4a886675 --- /dev/null +++ b/renderer/viewer/three/itemMesh.ts @@ -0,0 +1,427 @@ +import * as THREE from 'three' + +export interface Create3DItemMeshOptions { + depth?: number + pixelSize?: number +} + +export interface Create3DItemMeshResult { + geometry: THREE.BufferGeometry + totalVertices: number + totalTriangles: number +} + +/** + * Creates a 3D item geometry with front/back faces and connecting edges + * from a canvas containing the item texture + */ +export function create3DItemMesh ( + canvas: HTMLCanvasElement, + options: Create3DItemMeshOptions = {} +): Create3DItemMeshResult { + const { depth = 0.03, pixelSize } = options + + // Validate canvas dimensions + if (canvas.width <= 0 || canvas.height <= 0) { + throw new Error(`Invalid canvas dimensions: ${canvas.width}x${canvas.height}`) + } + + const ctx = canvas.getContext('2d')! + const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height) + const { data } = imageData + + const w = canvas.width + const h = canvas.height + const halfDepth = depth / 2 + const actualPixelSize = pixelSize ?? (1 / Math.max(w, h)) + + // Find opaque pixels + const isOpaque = (x: number, y: number) => { + if (x < 0 || y < 0 || x >= w || y >= h) return false + const i = (y * w + x) * 4 + return data[i + 3] > 128 // alpha > 128 + } + + const vertices: number[] = [] + const indices: number[] = [] + const uvs: number[] = [] + const normals: number[] = [] + + let vertexIndex = 0 + + // Helper to add a vertex + const addVertex = (x: number, y: number, z: number, u: number, v: number, nx: number, ny: number, nz: number) => { + vertices.push(x, y, z) + uvs.push(u, v) + normals.push(nx, ny, nz) + return vertexIndex++ + } + + // Helper to add a quad (two triangles) + const addQuad = (v0: number, v1: number, v2: number, v3: number) => { + indices.push(v0, v1, v2, v0, v2, v3) + } + + // Convert pixel coordinates to world coordinates + const pixelToWorld = (px: number, py: number) => { + const x = (px / w - 0.5) * actualPixelSize * w + const y = -(py / h - 0.5) * actualPixelSize * h + return { x, y } + } + + // Create a grid of vertices for front and back faces + const frontVertices: Array> = Array.from({ length: h + 1 }, () => Array.from({ length: w + 1 }, () => null)) + const backVertices: Array> = Array.from({ length: h + 1 }, () => Array.from({ length: w + 1 }, () => null)) + + // Create vertices at pixel corners + for (let py = 0; py <= h; py++) { + for (let px = 0; px <= w; px++) { + const { x, y } = pixelToWorld(px - 0.5, py - 0.5) + + // UV coordinates should map to the texture space of the extracted tile + const u = px / w + const v = py / h + + // Check if this vertex is needed for any face or edge + let needVertex = false + + // Check all 4 adjacent pixels to see if any are opaque + const adjacentPixels = [ + [px - 1, py - 1], // top-left pixel + [px, py - 1], // top-right pixel + [px - 1, py], // bottom-left pixel + [px, py] // bottom-right pixel + ] + + for (const [adjX, adjY] of adjacentPixels) { + if (isOpaque(adjX, adjY)) { + needVertex = true + break + } + } + + if (needVertex) { + frontVertices[py][px] = addVertex(x, y, halfDepth, u, v, 0, 0, 1) + backVertices[py][px] = addVertex(x, y, -halfDepth, u, v, 0, 0, -1) + } + } + } + + // Create front and back faces + for (let py = 0; py < h; py++) { + for (let px = 0; px < w; px++) { + if (!isOpaque(px, py)) continue + + const v00 = frontVertices[py][px] + const v10 = frontVertices[py][px + 1] + const v11 = frontVertices[py + 1][px + 1] + const v01 = frontVertices[py + 1][px] + + const b00 = backVertices[py][px] + const b10 = backVertices[py][px + 1] + const b11 = backVertices[py + 1][px + 1] + const b01 = backVertices[py + 1][px] + + if (v00 !== null && v10 !== null && v11 !== null && v01 !== null) { + // Front face + addQuad(v00, v10, v11, v01) + } + + if (b00 !== null && b10 !== null && b11 !== null && b01 !== null) { + // Back face (reversed winding) + addQuad(b10, b00, b01, b11) + } + } + } + + // Create edge faces for each side of the pixel with proper UVs + for (let py = 0; py < h; py++) { + for (let px = 0; px < w; px++) { + if (!isOpaque(px, py)) continue + + const pixelU = (px + 0.5) / w // Center of current pixel + const pixelV = (py + 0.5) / h + + // Left edge (x = px) + if (!isOpaque(px - 1, py)) { + const f0 = frontVertices[py][px] + const f1 = frontVertices[py + 1][px] + const b0 = backVertices[py][px] + const b1 = backVertices[py + 1][px] + + if (f0 !== null && f1 !== null && b0 !== null && b1 !== null) { + // Create new vertices for edge with current pixel's UV + const ef0 = addVertex(vertices[f0 * 3], vertices[f0 * 3 + 1], vertices[f0 * 3 + 2], pixelU, pixelV, -1, 0, 0) + const ef1 = addVertex(vertices[f1 * 3], vertices[f1 * 3 + 1], vertices[f1 * 3 + 2], pixelU, pixelV, -1, 0, 0) + const eb1 = addVertex(vertices[b1 * 3], vertices[b1 * 3 + 1], vertices[b1 * 3 + 2], pixelU, pixelV, -1, 0, 0) + const eb0 = addVertex(vertices[b0 * 3], vertices[b0 * 3 + 1], vertices[b0 * 3 + 2], pixelU, pixelV, -1, 0, 0) + addQuad(ef0, ef1, eb1, eb0) + } + } + + // Right edge (x = px + 1) + if (!isOpaque(px + 1, py)) { + const f0 = frontVertices[py + 1][px + 1] + const f1 = frontVertices[py][px + 1] + const b0 = backVertices[py + 1][px + 1] + const b1 = backVertices[py][px + 1] + + if (f0 !== null && f1 !== null && b0 !== null && b1 !== null) { + const ef0 = addVertex(vertices[f0 * 3], vertices[f0 * 3 + 1], vertices[f0 * 3 + 2], pixelU, pixelV, 1, 0, 0) + const ef1 = addVertex(vertices[f1 * 3], vertices[f1 * 3 + 1], vertices[f1 * 3 + 2], pixelU, pixelV, 1, 0, 0) + const eb1 = addVertex(vertices[b1 * 3], vertices[b1 * 3 + 1], vertices[b1 * 3 + 2], pixelU, pixelV, 1, 0, 0) + const eb0 = addVertex(vertices[b0 * 3], vertices[b0 * 3 + 1], vertices[b0 * 3 + 2], pixelU, pixelV, 1, 0, 0) + addQuad(ef0, ef1, eb1, eb0) + } + } + + // Top edge (y = py) + if (!isOpaque(px, py - 1)) { + const f0 = frontVertices[py][px] + const f1 = frontVertices[py][px + 1] + const b0 = backVertices[py][px] + const b1 = backVertices[py][px + 1] + + if (f0 !== null && f1 !== null && b0 !== null && b1 !== null) { + const ef0 = addVertex(vertices[f0 * 3], vertices[f0 * 3 + 1], vertices[f0 * 3 + 2], pixelU, pixelV, 0, -1, 0) + const ef1 = addVertex(vertices[f1 * 3], vertices[f1 * 3 + 1], vertices[f1 * 3 + 2], pixelU, pixelV, 0, -1, 0) + const eb1 = addVertex(vertices[b1 * 3], vertices[b1 * 3 + 1], vertices[b1 * 3 + 2], pixelU, pixelV, 0, -1, 0) + const eb0 = addVertex(vertices[b0 * 3], vertices[b0 * 3 + 1], vertices[b0 * 3 + 2], pixelU, pixelV, 0, -1, 0) + addQuad(ef0, ef1, eb1, eb0) + } + } + + // Bottom edge (y = py + 1) + if (!isOpaque(px, py + 1)) { + const f0 = frontVertices[py + 1][px + 1] + const f1 = frontVertices[py + 1][px] + const b0 = backVertices[py + 1][px + 1] + const b1 = backVertices[py + 1][px] + + if (f0 !== null && f1 !== null && b0 !== null && b1 !== null) { + const ef0 = addVertex(vertices[f0 * 3], vertices[f0 * 3 + 1], vertices[f0 * 3 + 2], pixelU, pixelV, 0, 1, 0) + const ef1 = addVertex(vertices[f1 * 3], vertices[f1 * 3 + 1], vertices[f1 * 3 + 2], pixelU, pixelV, 0, 1, 0) + const eb1 = addVertex(vertices[b1 * 3], vertices[b1 * 3 + 1], vertices[b1 * 3 + 2], pixelU, pixelV, 0, 1, 0) + const eb0 = addVertex(vertices[b0 * 3], vertices[b0 * 3 + 1], vertices[b0 * 3 + 2], pixelU, pixelV, 0, 1, 0) + addQuad(ef0, ef1, eb1, eb0) + } + } + } + } + + const geometry = new THREE.BufferGeometry() + geometry.setAttribute('position', new THREE.Float32BufferAttribute(vertices, 3)) + geometry.setAttribute('uv', new THREE.Float32BufferAttribute(uvs, 2)) + geometry.setAttribute('normal', new THREE.Float32BufferAttribute(normals, 3)) + geometry.setIndex(indices) + + // Compute normals properly + geometry.computeVertexNormals() + + return { + geometry, + totalVertices: vertexIndex, + totalTriangles: indices.length / 3 + } +} + +export interface ItemTextureInfo { + u: number + v: number + sizeX: number + sizeY: number +} + +export interface ItemMeshResult { + mesh: THREE.Object3D + itemsTexture?: THREE.Texture + itemsTextureFlipped?: THREE.Texture + cleanup?: () => void +} + +/** + * Extracts item texture region to a canvas + */ +export function extractItemTextureToCanvas ( + sourceTexture: THREE.Texture, + textureInfo: ItemTextureInfo +): HTMLCanvasElement { + const { u, v, sizeX, sizeY } = textureInfo + + // Calculate canvas size - fix the calculation + const canvasWidth = Math.max(1, Math.floor(sizeX * sourceTexture.image.width)) + const canvasHeight = Math.max(1, Math.floor(sizeY * sourceTexture.image.height)) + + const canvas = document.createElement('canvas') + canvas.width = canvasWidth + canvas.height = canvasHeight + + const ctx = canvas.getContext('2d')! + ctx.imageSmoothingEnabled = false + + // Draw the item texture region to canvas + ctx.drawImage( + sourceTexture.image, + u * sourceTexture.image.width, + v * sourceTexture.image.height, + sizeX * sourceTexture.image.width, + sizeY * sourceTexture.image.height, + 0, + 0, + canvas.width, + canvas.height + ) + + return canvas +} + +/** + * Creates either a 2D or 3D item mesh based on parameters + */ +export function createItemMesh ( + sourceTexture: THREE.Texture, + textureInfo: ItemTextureInfo, + options: { + faceCamera?: boolean + use3D?: boolean + depth?: number + } = {} +): ItemMeshResult { + const { faceCamera = false, use3D = true, depth = 0.03 } = options + const { u, v, sizeX, sizeY } = textureInfo + + if (faceCamera) { + // Create sprite for camera-facing items + const itemsTexture = sourceTexture.clone() + itemsTexture.flipY = true + itemsTexture.offset.set(u, 1 - v - sizeY) + itemsTexture.repeat.set(sizeX, sizeY) + itemsTexture.needsUpdate = true + itemsTexture.magFilter = THREE.NearestFilter + itemsTexture.minFilter = THREE.NearestFilter + + const spriteMat = new THREE.SpriteMaterial({ + map: itemsTexture, + transparent: true, + alphaTest: 0.1, + }) + const mesh = new THREE.Sprite(spriteMat) + + return { + mesh, + itemsTexture, + cleanup () { + itemsTexture.dispose() + } + } + } + + if (use3D) { + // Try to create 3D mesh + try { + const canvas = extractItemTextureToCanvas(sourceTexture, textureInfo) + const { geometry } = create3DItemMesh(canvas, { depth }) + + // Create texture from canvas for the 3D mesh + const itemsTexture = new THREE.CanvasTexture(canvas) + itemsTexture.magFilter = THREE.NearestFilter + itemsTexture.minFilter = THREE.NearestFilter + itemsTexture.wrapS = itemsTexture.wrapT = THREE.ClampToEdgeWrapping + itemsTexture.flipY = false + itemsTexture.needsUpdate = true + + const material = new THREE.MeshStandardMaterial({ + map: itemsTexture, + side: THREE.DoubleSide, + transparent: true, + alphaTest: 0.1, + }) + + const mesh = new THREE.Mesh(geometry, material) + + return { + mesh, + itemsTexture, + cleanup () { + itemsTexture.dispose() + geometry.dispose() + if (material.map) material.map.dispose() + material.dispose() + } + } + } catch (error) { + console.warn('Failed to create 3D item mesh, falling back to 2D:', error) + // Fall through to 2D rendering + } + } + + // Fallback to 2D flat rendering + const itemsTexture = sourceTexture.clone() + itemsTexture.flipY = true + itemsTexture.offset.set(u, 1 - v - sizeY) + itemsTexture.repeat.set(sizeX, sizeY) + itemsTexture.needsUpdate = true + itemsTexture.magFilter = THREE.NearestFilter + itemsTexture.minFilter = THREE.NearestFilter + + const itemsTextureFlipped = itemsTexture.clone() + itemsTextureFlipped.repeat.x *= -1 + itemsTextureFlipped.needsUpdate = true + itemsTextureFlipped.offset.set(u + sizeX, 1 - v - sizeY) + + const material = new THREE.MeshStandardMaterial({ + map: itemsTexture, + transparent: true, + alphaTest: 0.1, + }) + const materialFlipped = new THREE.MeshStandardMaterial({ + map: itemsTextureFlipped, + transparent: true, + alphaTest: 0.1, + }) + const mesh = new THREE.Mesh(new THREE.BoxGeometry(1, 1, 0), [ + new THREE.MeshBasicMaterial({ color: 0x00_00_00 }), new THREE.MeshBasicMaterial({ color: 0x00_00_00 }), + new THREE.MeshBasicMaterial({ color: 0x00_00_00 }), new THREE.MeshBasicMaterial({ color: 0x00_00_00 }), + material, materialFlipped, + ]) + + return { + mesh, + itemsTexture, + itemsTextureFlipped, + cleanup () { + itemsTexture.dispose() + itemsTextureFlipped.dispose() + material.dispose() + materialFlipped.dispose() + } + } +} + +/** + * Creates a complete 3D item mesh from a canvas texture + */ +export function createItemMeshFromCanvas ( + canvas: HTMLCanvasElement, + options: Create3DItemMeshOptions = {} +): THREE.Mesh { + const { geometry } = create3DItemMesh(canvas, options) + + // Base color texture for the item + const colorTexture = new THREE.CanvasTexture(canvas) + colorTexture.magFilter = THREE.NearestFilter + colorTexture.minFilter = THREE.NearestFilter + colorTexture.wrapS = colorTexture.wrapT = THREE.ClampToEdgeWrapping + colorTexture.flipY = false // Important for canvas textures + colorTexture.needsUpdate = true + + // Material - no transparency, no alpha test needed for edges + const material = new THREE.MeshBasicMaterial({ + map: colorTexture, + side: THREE.DoubleSide, + transparent: true, + alphaTest: 0.1 + }) + + return new THREE.Mesh(geometry, material) +}