From 563f5fa0071566313aedfd74926c6eba984e1362 Mon Sep 17 00:00:00 2001 From: Vitaly Date: Mon, 24 Mar 2025 20:17:54 +0300 Subject: [PATCH] feat: Add videos/images from source with protocol extension (#301) --- renderer/viewer/lib/worldrendererThree.ts | 411 ++++++++++++++++++++++ src/customChannels.ts | 282 ++++++++++++++- src/mineflayer/plugins/mouse.ts | 7 + src/optionsStorage.ts | 1 + src/react/GameInteractionOverlay.tsx | 3 +- src/react/TouchInteractionHint.tsx | 16 +- 6 files changed, 712 insertions(+), 8 deletions(-) diff --git a/renderer/viewer/lib/worldrendererThree.ts b/renderer/viewer/lib/worldrendererThree.ts index 697717c9..c181d43c 100644 --- a/renderer/viewer/lib/worldrendererThree.ts +++ b/renderer/viewer/lib/worldrendererThree.ts @@ -16,6 +16,18 @@ import { IPlayerState } from './basePlayerState' import { getMesh } from './entity/EntityMesh' import { armorModel } from './entity/armorModels' +interface MediaProperties { + position: { x: number, y: number, z: number } + size: { width: number, height: number } + src: string + rotation?: 0 | 1 | 2 | 3 // 0-3 for 0°, 90°, 180°, 270° + doubleSide?: boolean + background?: number // Hexadecimal color (e.g., 0x000000 for black) + opacity?: number // 0-1 value for transparency + uvMapping?: { startU: number, endU: number, startV: number, endV: number } + allowOrigins?: string[] | boolean +} + export class WorldRendererThree extends WorldRendererCommon { interactionLines: null | { blockPos; mesh } = null outputFormat = 'threeJs' as const @@ -28,6 +40,12 @@ export class WorldRendererThree extends WorldRendererCommon { holdingBlock: HoldingBlock holdingBlockLeft: HoldingBlock rendererDevice = '...' + customMedia = new Map void + }>() get tilesRendered () { return Object.values(this.sectionObjects).reduce((acc, obj) => acc + (obj as any).tilesCount, 0) @@ -457,6 +475,399 @@ export class WorldRendererThree extends WorldRendererCommon { console.warn('Failed to get renderer info', err) } } + + private createErrorTexture (width: number, height: number, background = 0x00_00_00): THREE.CanvasTexture { + const canvas = document.createElement('canvas') + // Scale up the canvas size for better text quality + canvas.width = width * 100 + canvas.height = height * 100 + + const ctx = canvas.getContext('2d') + if (!ctx) return new THREE.CanvasTexture(canvas) + + // Clear with transparent background + ctx.clearRect(0, 0, canvas.width, canvas.height) + + // Add background color + ctx.fillStyle = `rgba(${background >> 16 & 255}, ${background >> 8 & 255}, ${background & 255}, 0.5)` + ctx.fillRect(0, 0, canvas.width, canvas.height) + + // Add red text + ctx.fillStyle = '#ff0000' + ctx.font = 'bold 10px sans-serif' + ctx.textAlign = 'center' + ctx.textBaseline = 'middle' + ctx.fillText('Failed to load', canvas.width / 2, canvas.height / 2, canvas.width) + + const texture = new THREE.CanvasTexture(canvas) + texture.minFilter = THREE.LinearFilter + texture.magFilter = THREE.LinearFilter + return texture + } + + private createBackgroundTexture (width: number, height: number, color = 0x00_00_00, opacity = 1): THREE.CanvasTexture { + const canvas = document.createElement('canvas') + canvas.width = 1 + canvas.height = 1 + + const ctx = canvas.getContext('2d') + if (!ctx) return new THREE.CanvasTexture(canvas) + + // Convert hex color to rgba + const r = (color >> 16) & 255 + const g = (color >> 8) & 255 + const b = color & 255 + + ctx.fillStyle = `rgba(${r}, ${g}, ${b}, ${opacity})` + ctx.fillRect(0, 0, 1, 1) + + const texture = new THREE.CanvasTexture(canvas) + texture.minFilter = THREE.NearestFilter + texture.magFilter = THREE.NearestFilter + return texture + } + + validateOrigin (src: string, allowOrigins: string[] | boolean) { + if (allowOrigins === true) return true + if (allowOrigins === false) return false + const url = new URL(src) + return allowOrigins.some(origin => url.origin.endsWith(origin)) + } + + addMedia (id: string, props: MediaProperties) { + this.destroyMedia(id) + + const originSecurityError = props.allowOrigins !== undefined && !this.validateOrigin(props.src, props.allowOrigins) + if (originSecurityError) { + console.warn('Remote resource blocked due to security policy', props.src, 'allowed origins:', props.allowOrigins, 'you can control it with `remoteContentNotSameOrigin` option') + props.src = '' + } + + const isImage = props.src.endsWith('.png') || props.src.endsWith('.jpg') || props.src.endsWith('.jpeg') + + let video: HTMLVideoElement | undefined + if (!isImage) { + video = document.createElement('video') + video.src = props.src + video.loop = true + video.muted = true + video.playsInline = true + video.crossOrigin = 'anonymous' + } + + // Create background texture first + const backgroundTexture = this.createBackgroundTexture( + props.size.width, + props.size.height, + props.background, + // props.opacity ?? 1 + ) + + const handleError = () => { + const errorTexture = this.createErrorTexture(props.size.width, props.size.height, props.background) + material.map = errorTexture + material.needsUpdate = true + } + + // Create a plane geometry with configurable UV mapping + const geometry = new THREE.PlaneGeometry(1, 1) + + // Create material with initial properties using background texture + const material = new THREE.MeshBasicMaterial({ + map: backgroundTexture, + transparent: true, + side: props.doubleSide ? THREE.DoubleSide : THREE.FrontSide + }) + + const texture = video + ? new THREE.VideoTexture(video) + : new THREE.TextureLoader().load(props.src, () => { + if (this.customMedia.get(id)?.texture === texture) { + material.map = texture + material.needsUpdate = true + } + }, undefined, handleError) // todo cache + texture.minFilter = THREE.NearestFilter + texture.magFilter = THREE.NearestFilter + texture.format = THREE.RGBAFormat + texture.generateMipmaps = false + + // Create inner mesh for offsets + const mesh = new THREE.Mesh(geometry, material) + + const { mesh: panel } = this.positionMeshExact(mesh, THREE.MathUtils.degToRad((props.rotation ?? 0) * 90), props.position, props.size.width, props.size.height) + + this.scene.add(panel) + + if (video) { + // Start playing the video + video.play().catch(err => { + console.error('Failed to play video:', err) + handleError() + }) + + // Update texture in animation loop + mesh.onBeforeRender = () => { + if (video.readyState === video.HAVE_ENOUGH_DATA) { + if (material.map !== texture) { + material.map = texture + material.needsUpdate = true + } + texture.needsUpdate = true + } + } + } + + // UV mapping configuration + const updateUVMapping = (config: { startU: number, endU: number, startV: number, endV: number }) => { + const uvs = geometry.attributes.uv.array as Float32Array + uvs[0] = config.startU + uvs[1] = config.startV + uvs[2] = config.endU + uvs[3] = config.startV + uvs[4] = config.endU + uvs[5] = config.endV + uvs[6] = config.startU + uvs[7] = config.endV + geometry.attributes.uv.needsUpdate = true + } + + // Apply initial UV mapping if provided + if (props.uvMapping) { + updateUVMapping(props.uvMapping) + } + + // Store video data + this.customMedia.set(id, { + mesh: panel, + video, + texture, + updateUVMapping + }) + + return id + } + + setVideoPlaying (id: string, playing: boolean) { + const videoData = this.customMedia.get(id) + if (videoData?.video) { + if (playing) { + videoData.video.play().catch(console.error) + } else { + videoData.video.pause() + } + } + } + + setVideoSeeking (id: string, seconds: number) { + const videoData = this.customMedia.get(id) + if (videoData?.video) { + videoData.video.currentTime = seconds + } + } + + setVideoVolume (id: string, volume: number) { + const videoData = this.customMedia.get(id) + if (videoData?.video) { + videoData.video.volume = volume + } + } + + setVideoSpeed (id: string, speed: number) { + const videoData = this.customMedia.get(id) + if (videoData?.video) { + videoData.video.playbackRate = speed + } + } + + destroyMedia (id: string) { + const media = this.customMedia.get(id) + if (media) { + if (media.video) { + media.video.pause() + media.video.src = '' + } + this.scene.remove(media.mesh) + media.texture.dispose() + + // Get the inner mesh from the group + const mesh = media.mesh.children[0] as THREE.Mesh + if (mesh) { + mesh.geometry.dispose() + if (mesh.material instanceof THREE.Material) { + mesh.material.dispose() + } + } + + this.customMedia.delete(id) + } + } + + /** + * Positions a mesh exactly at startPosition and extends it along the rotation direction + * with the specified width and height + * + * @param mesh The mesh to position + * @param rotation Rotation in radians (applied to Y axis) + * @param startPosition The exact starting position (corner) of the mesh + * @param width Width of the mesh + * @param height Height of the mesh + * @param depth Depth of the mesh (default: 1) + * @returns The positioned mesh for chaining + */ + positionMeshExact ( + mesh: THREE.Mesh, + rotation: number, + startPosition: { x: number, y: number, z: number }, + width: number, + height: number, + depth = 1 + ) { + // avoid z-fighting with the ground plane + if (rotation === 0) { + startPosition.z += 0.001 + } + if (rotation === Math.PI / 2) { + startPosition.x -= 0.001 + } + if (rotation === Math.PI) { + startPosition.z -= 0.001 + } + if (rotation === 3 * Math.PI / 2) { + startPosition.x += 0.001 + } + + // rotation normalize coordinates + if (rotation === 0) { + startPosition.z += 1 + } + if (rotation === Math.PI) { + startPosition.x += 1 + } + if (rotation === 3 * Math.PI / 2) { + startPosition.z += 1 + startPosition.x += 1 + } + + + // First, clean up any previous transformations + mesh.matrix.identity() + mesh.position.set(0, 0, 0) + mesh.rotation.set(0, 0, 0) + mesh.scale.set(1, 1, 1) + + // By default, PlaneGeometry creates a plane in the XY plane (facing +Z) + // We need to set up the proper orientation for our use case + // Rotate the plane to face the correct direction based on the rotation parameter + mesh.rotateY(rotation) + if (rotation === Math.PI / 2 || rotation === 3 * Math.PI / 2) { + mesh.rotateZ(-Math.PI) + mesh.rotateX(-Math.PI) + } + + // Scale it to the desired size + mesh.scale.set(width, height, depth) + + // For a PlaneGeometry, if we want the corner at the origin, we need to offset + // by half the dimensions after scaling + mesh.geometry.translate(0.5, 0.5, 0) + mesh.geometry.attributes.position.needsUpdate = true + + // Now place the mesh at the start position + mesh.position.set(startPosition.x, startPosition.y, startPosition.z) + + // Create a group to hold our mesh and markers + const debugGroup = new THREE.Group() + debugGroup.add(mesh) + + // Add a marker at the starting position (should be exactly at pos) + const startMarker = new THREE.Mesh( + new THREE.BoxGeometry(0.1, 0.1, 0.1), + new THREE.MeshBasicMaterial({ color: 0xff_00_00 }) + ) + startMarker.position.copy(new THREE.Vector3(startPosition.x, startPosition.y, startPosition.z)) + debugGroup.add(startMarker) + + // Add a marker at the end position (width units away in the rotated direction) + const endX = startPosition.x + Math.cos(rotation) * width + const endZ = startPosition.z + Math.sin(rotation) * width + const endYMarker = new THREE.Mesh( + new THREE.BoxGeometry(0.1, 0.1, 0.1), + new THREE.MeshBasicMaterial({ color: 0x00_00_ff }) + ) + endYMarker.position.set(startPosition.x, startPosition.y + height, startPosition.z) + debugGroup.add(endYMarker) + + // Add a marker at the width endpoint + const endWidthMarker = new THREE.Mesh( + new THREE.BoxGeometry(0.1, 0.1, 0.1), + new THREE.MeshBasicMaterial({ color: 0xff_ff_00 }) + ) + endWidthMarker.position.set(endX, startPosition.y, endZ) + debugGroup.add(endWidthMarker) + + // Add a marker at the corner diagonal endpoint (both width and height) + const endCornerMarker = new THREE.Mesh( + new THREE.BoxGeometry(0.1, 0.1, 0.1), + new THREE.MeshBasicMaterial({ color: 0xff_00_ff }) + ) + endCornerMarker.position.set(endX, startPosition.y + height, endZ) + debugGroup.add(endCornerMarker) + + // Also add a visual helper to show the rotation direction + const directionHelper = new THREE.ArrowHelper( + new THREE.Vector3(Math.cos(rotation), 0, Math.sin(rotation)), + new THREE.Vector3(startPosition.x, startPosition.y, startPosition.z), + 1, + 0xff_00_00 + ) + debugGroup.add(directionHelper) + + return { + mesh, + debugGroup + } + } + + createTestCanvasTexture () { + const canvas = document.createElement('canvas') + canvas.width = 100 + canvas.height = 100 + const ctx = canvas.getContext('2d') + if (!ctx) return null + ctx.font = '10px Arial' + ctx.fillStyle = 'red' + ctx.fillText('Hello World', 0, 10) // at + return new THREE.CanvasTexture(canvas) + } + + /** + * Creates a test mesh that demonstrates the exact positioning + */ + addTestMeshExact (rotationNum: number) { + const pos = window.cursorBlockRel().position + console.log('Creating exact positioned test mesh at:', pos) + + // Create a plane mesh with a wireframe to visualize boundaries + const plane = new THREE.Mesh( + new THREE.PlaneGeometry(1, 1), + new THREE.MeshBasicMaterial({ + // side: THREE.DoubleSide, + map: this.createTestCanvasTexture() + }) + ) + + const width = 2 + const height = 1 + const rotation = THREE.MathUtils.degToRad(rotationNum * 90) // 90 degrees in radians + + // Position the mesh exactly where we want it + const { debugGroup } = this.positionMeshExact(plane, rotation, pos, width, height) + + viewer.scene.add(debugGroup) + console.log('Exact test mesh added with dimensions:', width, height, 'and rotation:', rotation) + + } } class StarField { diff --git a/src/customChannels.ts b/src/customChannels.ts index ff0f8a32..ede62876 100644 --- a/src/customChannels.ts +++ b/src/customChannels.ts @@ -1,5 +1,9 @@ import { Vec3 } from 'vec3' +import PItem from 'prismarine-item' +import * as THREE from 'three' +import { WorldRendererThree } from '../renderer/viewer/lib/worldrendererThree' import { options } from './optionsStorage' +import { jeiCustomCategories } from './inventoryWindows' customEvents.on('mineflayerBotCreated', async () => { if (!options.customChannels) return @@ -9,6 +13,8 @@ customEvents.on('mineflayerBotCreated', async () => { }) }) registerBlockModelsChannel() + registerMediaChannels() + registeredJeiChannel() }) const registerBlockModelsChannel = () => { @@ -88,7 +94,7 @@ const registeredJeiChannel = () => { type: ['pstring', { countType: 'i16' }] }, { - name: 'categoryTitle', + name: '_categoryTitle', type: ['pstring', { countType: 'i16' }] }, { @@ -102,8 +108,280 @@ const registeredJeiChannel = () => { bot._client.on(CHANNEL_NAME as any, (data) => { const { id, categoryTitle, items } = data - // ... + if (items === '') { + // remove category + jeiCustomCategories.value = jeiCustomCategories.value.filter(x => x.id !== id) + return + } + const PrismarineItem = PItem(bot.version) + jeiCustomCategories.value.push({ + id, + categoryTitle, + items: JSON.parse(items).map(x => { + const itemString = x.itemName || x.item_name || x.item || x.itemId + const itemId = loadedData.itemsByName[itemString.replace('minecraft:', '')] + if (!itemId) { + console.warn(`Could not add item ${itemString} to JEI category ${categoryTitle} because it was not found`) + return null + } + // const item = new PrismarineItem(itemId.id, x.itemCount || x.item_count || x.count || 1, x.itemDamage || x.item_damage || x.damage || 0, x.itemNbt || x.item_nbt || x.nbt || null) + return PrismarineItem.fromNotch({ + ...x, + itemId: itemId.id, + }) + }) + }) }) console.debug(`registered custom channel ${CHANNEL_NAME} channel`) } + +const registerMediaChannels = () => { + // Media Add Channel + const ADD_CHANNEL = 'minecraft-web-client:media-add' + const addPacketStructure = [ + 'container', + [ + { name: 'id', type: ['pstring', { countType: 'i16' }] }, + { name: 'x', type: 'f32' }, + { name: 'y', type: 'f32' }, + { name: 'z', type: 'f32' }, + { name: 'width', type: 'f32' }, + { name: 'height', type: 'f32' }, + // N, 0 + // W, 3 + // S, 2 + // E, 1 + { name: 'rotation', type: 'i16' }, // 0: 0° - towards positive z, 1: 90° - positive x, 2: 180° - negative z, 3: 270° - negative x (3-6 is same but double side) + { name: 'source', type: ['pstring', { countType: 'i16' }] }, + { name: 'loop', type: 'bool' }, + { name: '_volume', type: 'f32' }, // 0 + { name: '_aspectRatioMode', type: 'i16' }, // 0 + { name: '_background', type: 'i16' }, // 0 + { name: '_opacity', type: 'i16' }, // 1 + { name: '_cropXStart', type: 'f32' }, // 0 + { name: '_cropYStart', type: 'f32' }, // 0 + { name: '_cropXEnd', type: 'f32' }, // 0 + { name: '_cropYEnd', type: 'f32' }, // 0 + ] + ] + + // Media Control Channels + const PLAY_CHANNEL = 'minecraft-web-client:media-play' + const PAUSE_CHANNEL = 'minecraft-web-client:media-pause' + const SEEK_CHANNEL = 'minecraft-web-client:media-seek' + const VOLUME_CHANNEL = 'minecraft-web-client:media-volume' + const SPEED_CHANNEL = 'minecraft-web-client:media-speed' + const DESTROY_CHANNEL = 'minecraft-web-client:media-destroy' + + const noDataPacketStructure = [ + 'container', + [ + { name: 'id', type: ['pstring', { countType: 'i16' }] } + ] + ] + + const setNumberPacketStructure = [ + 'container', + [ + { name: 'id', type: ['pstring', { countType: 'i16' }] }, + { name: 'seconds', type: 'f32' } + ] + ] + + // Register channels + bot._client.registerChannel(ADD_CHANNEL, addPacketStructure, true) + bot._client.registerChannel(PLAY_CHANNEL, noDataPacketStructure, true) + bot._client.registerChannel(PAUSE_CHANNEL, noDataPacketStructure, true) + bot._client.registerChannel(SEEK_CHANNEL, setNumberPacketStructure, true) + bot._client.registerChannel(VOLUME_CHANNEL, setNumberPacketStructure, true) + bot._client.registerChannel(SPEED_CHANNEL, setNumberPacketStructure, true) + bot._client.registerChannel(DESTROY_CHANNEL, noDataPacketStructure, true) + + // Handle media add + bot._client.on(ADD_CHANNEL as any, (data) => { + const { id, x, y, z, width, height, rotation, source, loop, background, opacity } = data + + const worldRenderer = viewer.world as WorldRendererThree + + // Destroy existing video if it exists + worldRenderer.destroyMedia(id) + + // Add new video + worldRenderer.addMedia(id, { + position: { x, y, z }, + size: { width, height }, + // side: 'towards', + src: source, + rotation: rotation as 0 | 1 | 2 | 3, + doubleSide: false, + background, + opacity: opacity / 100, + allowOrigins: options.remoteContentNotSameOrigin === false ? [getCurrentTopDomain()] : options.remoteContentNotSameOrigin + }) + + // Set loop state + if (!loop) { + const videoData = worldRenderer.customMedia.get(id) + if (videoData?.video) { + videoData.video.loop = false + } + } + }) + + // Handle media play + bot._client.on(PLAY_CHANNEL as any, (data) => { + const { id } = data + const worldRenderer = viewer.world as WorldRendererThree + worldRenderer.setVideoPlaying(id, true) + }) + + // Handle media pause + bot._client.on(PAUSE_CHANNEL as any, (data) => { + const { id } = data + const worldRenderer = viewer.world as WorldRendererThree + worldRenderer.setVideoPlaying(id, false) + }) + + // Handle media seek + bot._client.on(SEEK_CHANNEL as any, (data) => { + const { id, seconds } = data + const worldRenderer = viewer.world as WorldRendererThree + worldRenderer.setVideoSeeking(id, seconds) + }) + + // Handle media destroy + bot._client.on(DESTROY_CHANNEL as any, (data) => { + const { id } = data + const worldRenderer = viewer.world as WorldRendererThree + worldRenderer.destroyMedia(id) + }) + + // Handle media volume + bot._client.on(VOLUME_CHANNEL as any, (data) => { + const { id, volume } = data + const worldRenderer = viewer.world as WorldRendererThree + worldRenderer.setVideoVolume(id, volume) + }) + + // Handle media speed + bot._client.on(SPEED_CHANNEL as any, (data) => { + const { id, speed } = data + const worldRenderer = viewer.world as WorldRendererThree + worldRenderer.setVideoSpeed(id, speed) + }) + + // --- + + // Video interaction channel + const interactionPacketStructure = [ + 'container', + [ + { name: 'id', type: ['pstring', { countType: 'i16' }] }, + { name: 'x', type: 'f32' }, + { name: 'y', type: 'f32' }, + { name: 'isRightClick', type: 'bool' } + ] + ] + + bot._client.registerChannel(MEDIA_INTERACTION_CHANNEL, interactionPacketStructure, true) + + console.debug('Registered media channels') +} + +const MEDIA_INTERACTION_CHANNEL = 'minecraft-web-client:media-interaction' + +export const sendVideoInteraction = (id: string, x: number, y: number, isRightClick: boolean) => { + bot._client.writeChannel(MEDIA_INTERACTION_CHANNEL, { id, x, y, isRightClick }) +} + +export const videoCursorInteraction = () => { + const worldRenderer = viewer.world as WorldRendererThree + const { camera } = worldRenderer + const raycaster = new THREE.Raycaster() + + // Get mouse position at center of screen + const mouse = new THREE.Vector2(0, 0) + + // Update the raycaster + raycaster.setFromCamera(mouse, camera) + + // Check intersection with all video meshes + for (const [id, videoData] of worldRenderer.customMedia.entries()) { + // Get the actual mesh (first child of the group) + const mesh = videoData.mesh.children[0] as THREE.Mesh + if (!mesh) continue + + const intersects = raycaster.intersectObject(mesh, false) + if (intersects.length > 0) { + const intersection = intersects[0] + const { uv } = intersection + if (!uv) return null + + return { + id, + x: uv.x, + y: uv.y + } + } + } + + return null +} +window.videoCursorInteraction = videoCursorInteraction + +const addTestVideo = (rotation = 0 as 0 | 1 | 2 | 3, scale = 1, isImage = false) => { + const block = window.cursorBlockRel() + if (!block) return + const { position: startPosition } = block + + const worldRenderer = viewer.world as WorldRendererThree + + // Add video with proper positioning + worldRenderer.addMedia('test-video', { + position: { + x: startPosition.x, + y: startPosition.y + 1, + z: startPosition.z + }, + size: { + width: scale, + height: scale + }, + src: isImage ? 'https://bucket.mcraft.fun/test_image.png' : 'https://bucket.mcraft.fun/test_video.mp4', + rotation, + // doubleSide: true, + background: 0x00_00_00, // Black color + // TODO broken + // uvMapping: { + // startU: 0, + // endU: 1, + // startV: 0, + // endV: 1 + // }, + opacity: 1, + allowOrigins: true, + }) +} +window.addTestVideo = addTestVideo + +function getCurrentTopDomain (): string { + const { hostname } = location + // Split hostname into parts + const parts = hostname.split('.') + + // Handle special cases like co.uk, com.br, etc. + if (parts.length > 2) { + // Check for common country codes with additional segments + if (parts.at(-2) === 'co' || + parts.at(-2) === 'com' || + parts.at(-2) === 'org' || + parts.at(-2) === 'gov') { + // Return last 3 parts (e.g., example.co.uk) + return parts.slice(-3).join('.') + } + } + + // Return last 2 parts (e.g., example.com) + return parts.slice(-2).join('.') +} diff --git a/src/mineflayer/plugins/mouse.ts b/src/mineflayer/plugins/mouse.ts index e5b5e283..6841424f 100644 --- a/src/mineflayer/plugins/mouse.ts +++ b/src/mineflayer/plugins/mouse.ts @@ -22,6 +22,7 @@ import destroyStage9 from '../../../assets/destroy_stage_9.png' import { options } from '../../optionsStorage' import { isCypress } from '../../standaloneUtils' import { playerState } from '../playerState' +import { sendVideoInteraction, videoCursorInteraction } from '../../customChannels' function createDisplayManager (bot: Bot, scene: THREE.Scene, renderer: THREE.WebGLRenderer) { // State @@ -180,6 +181,12 @@ const domListeners = (bot: Bot) => { if (e.isTrusted && !document.pointerLockElement && !isCypress()) return if (!isGameActive(true)) return + const videoInteraction = videoCursorInteraction() + if (videoInteraction) { + sendVideoInteraction(videoInteraction.id, videoInteraction.x, videoInteraction.y, e.button === 0) + return + } + if (e.button === 0) { bot.leftClickStart() } else if (e.button === 2) { diff --git a/src/optionsStorage.ts b/src/optionsStorage.ts index e0de6d4a..b5025ab4 100644 --- a/src/optionsStorage.ts +++ b/src/optionsStorage.ts @@ -57,6 +57,7 @@ const defaultOptions = { packetsLoggerPreset: 'all' as 'all' | 'no-buffers', serversAutoVersionSelect: 'auto' as 'auto' | 'latest' | '1.20.4' | string, customChannels: false, + remoteContentNotSameOrigin: false as boolean | string[], packetsReplayAutoStart: false, preciseMouseInput: false, // todo ui setting, maybe enable by default? diff --git a/src/react/GameInteractionOverlay.tsx b/src/react/GameInteractionOverlay.tsx index 7c406e8d..90f7655c 100644 --- a/src/react/GameInteractionOverlay.tsx +++ b/src/react/GameInteractionOverlay.tsx @@ -5,6 +5,7 @@ import { options } from '../optionsStorage' import { activeModalStack, isGameActive, miscUiState } from '../globalState' import { onCameraMove, CameraMoveEvent } from '../cameraRotationControls' import { pointerLock, isInRealGameSession } from '../utils' +import { videoCursorInteraction } from '../customChannels' import { handleMovementStickDelta, joystickPointer } from './TouchAreasControls' /** after what time of holding the finger start breaking the block */ @@ -153,7 +154,7 @@ function GameInteractionOverlayInner ({ // single click action const MOUSE_BUTTON_RIGHT = 2 const MOUSE_BUTTON_LEFT = 0 - const gonnaAttack = !!bot.mouse.getCursorState().entity + const gonnaAttack = !!bot.mouse.getCursorState().entity || !!videoCursorInteraction() document.dispatchEvent(new MouseEvent('mousedown', { button: gonnaAttack ? MOUSE_BUTTON_LEFT : MOUSE_BUTTON_RIGHT })) bot.mouse.update() document.dispatchEvent(new MouseEvent('mouseup', { button: gonnaAttack ? MOUSE_BUTTON_LEFT : MOUSE_BUTTON_RIGHT })) diff --git a/src/react/TouchInteractionHint.tsx b/src/react/TouchInteractionHint.tsx index d4bc3496..e08e4634 100644 --- a/src/react/TouchInteractionHint.tsx +++ b/src/react/TouchInteractionHint.tsx @@ -2,6 +2,7 @@ import { useEffect, useState } from 'react' import { useSnapshot } from 'valtio' import { options } from '../optionsStorage' import { activeModalStack } from '../globalState' +import { videoCursorInteraction } from '../customChannels' import PixelartIcon, { pixelartIcons } from './PixelartIcon' import styles from './TouchInteractionHint.module.css' import { useUsingTouch } from './utilsApp' @@ -14,12 +15,17 @@ export default () => { useEffect(() => { const update = () => { - const cursorState = bot.mouse.getCursorState() - if (cursorState.entity) { - const entityName = cursorState.entity.displayName ?? cursorState.entity.name - setHintText(`Attack ${entityName}`) + const videoInteraction = videoCursorInteraction() + if (videoInteraction) { + setHintText(`Interact with video`) } else { - setHintText(null) + const cursorState = bot.mouse.getCursorState() + if (cursorState.entity) { + const entityName = cursorState.entity.displayName ?? cursorState.entity.name + setHintText(`Attack ${entityName}`) + } else { + setHintText(null) + } } }