import { createScopedThreejs } from 'threejs-miniprogram' import { registerGLTFLoader } from '../loaders/gltf-loader' const threeBehavior = Behavior({ methods: { // 针对 threejs 的初始化逻辑 initTHREE() { const THREE = this.THREE = createScopedThreejs(this.canvas) registerGLTFLoader(THREE) // glTF loader this.loader = new this.THREE.GLTFLoader() // 相机 this.camera = new THREE.PerspectiveCamera(50, 0.7, 0.1, 1000); // 场景 const scene = this.scene = new THREE.Scene() const sceneCull = this.sceneCull = new THREE.Scene() // 光源 const ambientLight = new THREE.AmbientLight( 0x555555 ); // 氛围光 scene.add( ambientLight ); const dirLight = new THREE.DirectionalLight(0xffffff, 1) // 平行光 dirLight.position.set(1, 1, 1); scene.add(dirLight) const ambientLightCull = new THREE.AmbientLight( 0x555555 ); // 氛围光 sceneCull.add( ambientLightCull ); const dirLightCull = new THREE.DirectionalLight(0xffffff, 1) // 平行光 dirLightCull.position.set(1, 1, 1); sceneCull.add(dirLightCull) // 渲染层 const renderer = this.renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true }) renderer.gammaOutput = true renderer.gammaFactor = 2.2 }, initYUVShader() { const gl = this.gl = this.renderer.getContext() const currentProgram = gl.getParameter(gl.CURRENT_PROGRAM) const vs = ` attribute vec2 a_position; attribute vec2 a_texCoord; uniform mat3 displayTransform; varying vec2 v_texCoord; void main() { vec3 p = displayTransform * vec3(a_position, 0); gl_Position = vec4(p, 1); v_texCoord = a_texCoord; } ` const fs = ` precision highp float; uniform sampler2D y_texture; uniform sampler2D uv_texture; varying vec2 v_texCoord; void main() { vec4 y_color = texture2D(y_texture, v_texCoord); vec4 uv_color = texture2D(uv_texture, v_texCoord); float Y, U, V; float R ,G, B; Y = y_color.r; U = uv_color.r - 0.5; V = uv_color.a - 0.5; R = Y + 1.402 * V; G = Y - 0.344 * U - 0.714 * V; B = Y + 1.772 * U; gl_FragColor = vec4(R, G, B, 1.0); } ` const vertShader = gl.createShader(gl.VERTEX_SHADER) gl.shaderSource(vertShader, vs) gl.compileShader(vertShader) const fragShader = gl.createShader(gl.FRAGMENT_SHADER) gl.shaderSource(fragShader, fs) gl.compileShader(fragShader) const program = this._program = gl.createProgram() this._program.gl = gl gl.attachShader(program, vertShader) gl.attachShader(program, fragShader) gl.deleteShader(vertShader) gl.deleteShader(fragShader) gl.linkProgram(program) gl.useProgram(program) const uniformYTexture = gl.getUniformLocation(program, 'y_texture') gl.uniform1i(uniformYTexture, 5) const uniformUVTexture = gl.getUniformLocation(program, 'uv_texture') gl.uniform1i(uniformUVTexture, 6) this._dt = gl.getUniformLocation(program, 'displayTransform') gl.useProgram(currentProgram) }, initVAO(program) { const gl = this.renderer.getContext() const ext = gl.getExtension('OES_vertex_array_object') this.ext = ext const currentVAO = gl.getParameter(gl.VERTEX_ARRAY_BINDING) const vao = ext.createVertexArrayOES() ext.bindVertexArrayOES(vao) const posAttr = gl.getAttribLocation(program, 'a_position') const pos = gl.createBuffer() gl.bindBuffer(gl.ARRAY_BUFFER, pos) gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([1, 1, -1, 1, 1, -1, -1, -1]), gl.STATIC_DRAW) gl.vertexAttribPointer(posAttr, 2, gl.FLOAT, false, 0, 0) gl.enableVertexAttribArray(posAttr) vao.posBuffer = pos const texcoordAttr = gl.getAttribLocation(program, 'a_texCoord') const texcoord = gl.createBuffer() gl.bindBuffer(gl.ARRAY_BUFFER, texcoord) gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([1, 1, 0, 1, 1, 0, 0, 0]), gl.STATIC_DRAW) gl.vertexAttribPointer(texcoordAttr, 2, gl.FLOAT, false, 0, 0) gl.enableVertexAttribArray(texcoordAttr) vao.texcoordBuffer = texcoord ext.bindVertexArrayOES(currentVAO) return vao; }, initYUV() { this.initYUVShader() this._vao = this.initVAO(this._program); }, renderYUV(frame) { const gl = this.renderer.getContext() gl.disable(gl.DEPTH_TEST) const { yTexture, uvTexture } = frame.getCameraTexture(gl, 'yuv') const displayTransform = frame.getDisplayTransform() if (yTexture && uvTexture) { const currentProgram = gl.getParameter(gl.CURRENT_PROGRAM) const currentActiveTexture = gl.getParameter(gl.ACTIVE_TEXTURE) const currentVAO = gl.getParameter(gl.VERTEX_ARRAY_BINDING) gl.useProgram(this._program) this.ext.bindVertexArrayOES(this._vao) gl.uniformMatrix3fv(this._dt, false, displayTransform) gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1) gl.activeTexture(gl.TEXTURE0 + 5) const bindingTexture5 = gl.getParameter(gl.TEXTURE_BINDING_2D) gl.bindTexture(gl.TEXTURE_2D, yTexture) gl.activeTexture(gl.TEXTURE0 + 6) const bindingTexture6 = gl.getParameter(gl.TEXTURE_BINDING_2D) gl.bindTexture(gl.TEXTURE_2D, uvTexture) gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4) gl.bindTexture(gl.TEXTURE_2D, bindingTexture6) gl.activeTexture(gl.TEXTURE0 + 5) gl.bindTexture(gl.TEXTURE_2D, bindingTexture5) gl.useProgram(currentProgram) gl.activeTexture(currentActiveTexture) this.ext.bindVertexArrayOES(currentVAO) } }, initDepthShaderHint() { const gl = this.gl = this.renderer.getContext() const ext = gl.getExtension("OES_texture_float"); if (!ext) console.warn('OES_texture_float not support'); const currentProgram = gl.getParameter(gl.CURRENT_PROGRAM) const vs = ` precision highp float; attribute vec2 a_position; attribute vec2 a_texCoord; uniform mat3 displayTransform; varying vec2 v_texCoord; void main() { vec3 p = displayTransform * vec3(a_position, 0); gl_Position = vec4(p, 1); v_texCoord = a_texCoord; } ` const fs = ` precision highp float; uniform sampler2D depth_texture; varying vec2 v_texCoord; void main() { vec4 depth_color = texture2D(depth_texture, v_texCoord); gl_FragColor = vec4(depth_color.rgb, 1.0); } ` const vertShader = gl.createShader(gl.VERTEX_SHADER) gl.shaderSource(vertShader, vs) gl.compileShader(vertShader) const fragShader = gl.createShader(gl.FRAGMENT_SHADER) gl.shaderSource(fragShader, fs) gl.compileShader(fragShader) const program = this._depthProgram = gl.createProgram() this._depthProgram.gl = gl gl.attachShader(program, vertShader) gl.attachShader(program, fragShader) gl.deleteShader(vertShader) gl.deleteShader(fragShader) gl.linkProgram(program) gl.useProgram(program) const uniformTexture = gl.getUniformLocation(program, 'depth_texture') gl.uniform1i(uniformTexture, 5) this._depthDt = gl.getUniformLocation(program, 'displayTransform') gl.useProgram(currentProgram) }, initDepthVAOHint() { const gl = this.renderer.getContext() const ext = gl.getExtension('OES_vertex_array_object') this.ext = ext const currentVAO = gl.getParameter(gl.VERTEX_ARRAY_BINDING) const vao = ext.createVertexArrayOES() ext.bindVertexArrayOES(vao) const posAttr = gl.getAttribLocation(this._depthProgram, 'a_position') const pos = gl.createBuffer() gl.bindBuffer(gl.ARRAY_BUFFER, pos) gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([0.3, 0.3, 1, 0.3, 0.3, 1, 1, 1]), gl.STATIC_DRAW) gl.vertexAttribPointer(posAttr, 2, gl.FLOAT, false, 0, 0) gl.enableVertexAttribArray(posAttr) vao.posBuffer = pos const texcoordAttr = gl.getAttribLocation(this._depthProgram, 'a_texCoord') const texcoord = gl.createBuffer() gl.bindBuffer(gl.ARRAY_BUFFER, texcoord) gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([0, 0, 1, 0, 0, 1, 1, 1]), gl.STATIC_DRAW) gl.vertexAttribPointer(texcoordAttr, 2, gl.FLOAT, false, 0, 0) gl.enableVertexAttribArray(texcoordAttr) vao.texcoordBuffer = texcoord ext.bindVertexArrayOES(currentVAO) this._depthVao = vao }, initDepthShader() { const gl = this.gl = this.renderer.getContext() const currentProgram = gl.getParameter(gl.CURRENT_PROGRAM) const dvs = `#version 300 es precision highp float; in vec2 a_position; in vec2 a_texCoord; uniform mat3 displayTransform; uniform sampler2D depth_texture; out vec2 v_texCoord; void main() { vec3 p = displayTransform * vec3(a_position, 1); v_texCoord = a_texCoord; vec4 depth_color = texture(depth_texture, v_texCoord); gl_Position = vec4(p.x, p.y, p.z, 1); } ` const dfs = `#version 300 es precision highp float; uniform sampler2D depth_texture; out vec4 FragColor; in vec2 v_texCoord; void main() { vec4 depth_color = texture(depth_texture, v_texCoord); gl_FragDepth = depth_color.r; // FragColor = vec4(depth_color.rgb, 1.0); } ` const vertShader = gl.createShader(gl.VERTEX_SHADER) gl.shaderSource(vertShader, dvs) gl.compileShader(vertShader) const fragShader = gl.createShader(gl.FRAGMENT_SHADER) gl.shaderSource(fragShader, dfs) gl.compileShader(fragShader) const program = this._depthOutputProgram = gl.createProgram() this._depthOutputProgram.gl = gl gl.attachShader(program, vertShader) gl.attachShader(program, fragShader) gl.deleteShader(vertShader) gl.deleteShader(fragShader) gl.linkProgram(program) gl.useProgram(program) const uniformDepthTexture = gl.getUniformLocation(this._depthOutputProgram, 'depth_texture') gl.uniform1i(uniformDepthTexture, 5) gl.getUniformLocation(this._depthOutputProgram, 'displayTransform') gl.useProgram(currentProgram) }, initDepthGL(){ // 初始化提示 this.initDepthShaderHint() this.initDepthVAOHint() // 初始化深度纹理相关 this.initDepthShader(); this._vaoDepth = this.initVAO(this._depthOutputProgram); }, renderDepthGLHint(frame) { const gl = this.renderer.getContext(); const displayTransform = frame.getDisplayTransform() // DepthBuffer const depthBufferRes = frame.getDepthBuffer(); const depthBuffer = new Float32Array(depthBufferRes.DepthAddress); // console.log('depthBuffer', depthBuffer[0], depthBuffer[16], depthBuffer[16 * 16], depthBuffer[56 * 56]); const texture = gl.createTexture(); gl.bindTexture(gl.TEXTURE_2D, texture); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); const width = depthBufferRes.width; const height = depthBufferRes.height; // 先直接采用 uint8 写入深度纹理,使用浮点写入的方法会存在锯齿 const data = new Uint8Array(width * height * 4); for (let i = 0; i < depthBuffer.length; i++) { let num = parseInt(depthBuffer[i] * 255); data[i] = num; } gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, data); const currentProgram = gl.getParameter(gl.CURRENT_PROGRAM) const currentActiveTexture = gl.getParameter(gl.ACTIVE_TEXTURE) const currentVAO = gl.getParameter(gl.VERTEX_ARRAY_BINDING) gl.useProgram(this._depthProgram) this.ext.bindVertexArrayOES(this._depthVao) gl.uniformMatrix3fv(this._depthDt, false, displayTransform) gl.activeTexture(gl.TEXTURE0 + 5) const bindingTexture5 = gl.getParameter(gl.TEXTURE_BINDING_2D) gl.bindTexture(gl.TEXTURE_2D, texture) gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4) gl.activeTexture(gl.TEXTURE0 + 5) gl.bindTexture(gl.TEXTURE_2D, bindingTexture5) gl.useProgram(currentProgram) gl.activeTexture(currentActiveTexture) this.ext.bindVertexArrayOES(currentVAO) }, renderDepthGL(frame) { const gl = this.renderer.getContext() const displayTransform = frame.getDisplayTransform() // DepthBuffer const depthBufferRes = frame.getDepthBuffer(); const depthBuffer = new Float32Array(depthBufferRes.DepthAddress); const texture = gl.createTexture(); gl.bindTexture(gl.TEXTURE_2D, texture); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); const width = depthBufferRes.width; const height = depthBufferRes.height; // 先直接采用 uint8 写入深度纹理,使用浮点写入的方法会存在锯齿 const data = new Uint8Array(width * height * 4); for (let i = 0; i < depthBuffer.length; i++) { let num = parseInt(depthBuffer[i] * 255); data[i] = num; } gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, data); // console.log('gl depth texture end') // 绘制左下角提示 const currentProgram = gl.getParameter(gl.CURRENT_PROGRAM) const currentActiveTexture = gl.getParameter(gl.ACTIVE_TEXTURE) const currentVAO = gl.getParameter(gl.VERTEX_ARRAY_BINDING) const bindingTexture = gl.getParameter(gl.TEXTURE_BINDING_2D) gl.useProgram(this._depthProgram) this.ext.bindVertexArrayOES(this._depthVao) gl.uniformMatrix3fv(this._depthDt, false, displayTransform) gl.activeTexture(gl.TEXTURE0 + 5) const bindingTexture5 = gl.getParameter(gl.TEXTURE_BINDING_2D) gl.bindTexture(gl.TEXTURE_2D, texture) gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4) gl.activeTexture(gl.TEXTURE0 + 5) gl.bindTexture(gl.TEXTURE_2D, bindingTexture5) gl.useProgram(currentProgram) gl.activeTexture(currentActiveTexture) this.ext.bindVertexArrayOES(currentVAO) // console.log('gl hint end') // 写入深度遮挡纹理到深度值 gl.enable(gl.DEPTH_TEST) gl.depthMask(true) gl.depthFunc(gl.ALWAYS) this.ext.bindVertexArrayOES(this._vaoDepth) gl.useProgram(this._depthOutputProgram) gl.uniformMatrix3fv(this._depthDt, false, displayTransform) gl.activeTexture(gl.TEXTURE0 + 5) const bindingTexture5Depth = gl.getParameter(gl.TEXTURE_BINDING_2D) gl.bindTexture(gl.TEXTURE_2D, texture) gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4) gl.activeTexture(gl.TEXTURE0 + 5) gl.bindTexture(gl.TEXTURE_2D, bindingTexture5Depth) gl.useProgram(currentProgram) gl.activeTexture(currentActiveTexture) gl.bindTexture(gl.TEXTURE_2D, bindingTexture) this.ext.bindVertexArrayOES(currentVAO) gl.depthMask(false) gl.depthFunc(gl.LESS) // console.log('gl depth draw end') }, }, }) export default threeBehavior;