微信小程序xr-frame透明视频实现

在开发AR小程序的时候需要实现用到透明视频效果。这里使用MP4格式的视频。它的左侧为视频的rgb通道信息，右侧为动画alpha通道信息，左右两侧动画同步播放。

关于透明视频制作的方法：

之前在unity中使用过这种MP4视频，测试时放到小程序中也可以共用。

客户或美术提供带有alpha通道的MOV视频，再使用ffmpeg代码转换。或者AE处理也可以，但是个人感觉用代码更方便一些。

ffmpeg -i b.mov -vf "split [a], pad=iw*2:ih [b], [a] alphaextract, [b] overlay=w" -c:v libx264 -crf 18 -preset veryfast -pix_fmt yuv420p -movflags +faststart -y output-lr-mov.mp4

MOV导出时的注意事项：

• 使用 ProRes4444 这类支持透明度的编解码器
• 要选择"straight" alpha 模式（而不是 “premultiplied”）

小程序代码

Components/index.js

Component({
  behaviors: [require('../common/share-behavior').default],
  properties: {
    a: Number,
  },
  data: {
    loaded: false,
    arReady: false,
  },
  lifetimes: {
    async attached() {
      console.log('data', this.data)
      // Add the effect registration here
      const xrFrameSystem = wx.getXrFrameSystem();
      
      function createVideoTsbsEffect(scene) {
        return scene.createEffect({
          "name": "video-tsbs",
          "properties": [{
            "key": "u_baseColorFactor",
            "type": 3,
            "default": [1, 1, 1, 0]
          }],
          "images": [{
            "key": "u_baseColorMap",
            "default": "white",
            "macro": "WX_USE_BASECOLORMAP"
          }],
          "defaultRenderQueue": 3000,
          "passes": [{
            "renderStates": {
              "cullOn": false,
              "blendOn": true,
              "depthWrite": true,
              "cullFace": 2
            },
            "lightMode": "ForwardBase",
            "useMaterialRenderStates": true,
            "shaders": [0, 1]
          }],
          "shaders": [
            `#version 100
            precision highp float;
            attribute vec3 a_position;
            attribute vec2 a_texCoord;

            uniform mat4 u_projection;
            uniform mat4 u_world;
            uniform mat4 u_view;
            varying highp vec2 vTextureCoord;
            
            void main()
            {
              vTextureCoord = a_texCoord;
              gl_Position = u_projection * u_view * u_world * vec4(a_position,1.0);
            }`,
            `#version 100
            precision highp float;

            uniform highp vec4 u_baseColorFactor;
            #ifdef WX_USE_BASECOLORMAP
              uniform sampler2D u_baseColorMap;
            #endif

            varying highp vec2 vTextureCoord;

            void main()
            {
          #ifdef WX_USE_BASECOLORMAP
            vec4 color = texture2D(u_baseColorMap, vec2(vTextureCoord.x*0.5,vTextureCoord.y));
            vec4 colora = texture2D(u_baseColorMap, vec2(vTextureCoord.x*0.5 + 0.5,vTextureCoord.y));

              vec4 baseColor = vec4(color.xyz,colora.x);
          #else
              vec4 baseColor = u_baseColorFactor;
          #endif

            gl_FragData[0] = baseColor;
            }  
          `
          ]
        })
      }
      
      xrFrameSystem.registerEffect("video-tsbs", createVideoTsbsEffect);
    }
  },
  methods: {
    handleReady({detail}) {
      const xrScene = this.scene = detail.value;
      console.log('xr-scene', xrScene);
      
      
      this.triggerEvent('sceneReady', { value: xrScene });
    },
    handleAssetsProgress: function({detail}) {
      console.log('assets progress', detail.value);
    },
    handleAssetsLoaded: function({detail}) {
      console.log('assets loaded', detail.value);
      this.setData({loaded: true});
    },
    handleARReady: function({detail}) {
      console.log('arReady', this.scene.ar.arVersion);
    },
    
    showModel() {
      console.log('showModel called');
      if (this.scene) {
        
        const modelNode = this.scene.getNodeById('setitem');
        
        
        console.log('模型节点：', modelNode);
        
        if (modelNode) {
          console.log('找到模型，设置为可见');
          modelNode.visible = true;
        } else {
          
          const allNodes = this.scene.children;
          console.log('所有节点：', allNodes);
        }
      } else {
        console.log('场景未初始化');
      }
    }
    
  }
})

Components/index.wxml

这里要把src="你的视频.mp4"这里改为你的视频URL

<xr-scene ar-system="modes:threeDof" bind:ready="handleReady" bind:ar-ready="handleARReady">
  <xr-assets bind:progress="handleAssetsProgress" bind:loaded="handleAssetsLoaded">
    <xr-asset-load type="gltf" asset-id="gltf-item" src="https://mmbizwxaminiprogram-1258344707.cos.ap-guangzhou.myqcloud.com/xr-frame/demo/just_a_girl/index.glb" />
  </xr-assets>
  <xr-asset-load type="video-texture" asset-id="test" src="你的视频.mp4" options="autoPlay:true,loop:true" />
	<xr-asset-material asset-id="video-tsbs-mat" effect="video-tsbs" />
  <xr-node node-id="setitem" visible="false">
    <xr-mesh node-id="mesh-x" position="2 0 0" scale="2 0.02 0.02" geometry="cube" uniforms="u_baseColorFactor:1 0 0 1" ></xr-mesh>
    <xr-mesh node-id="mesh-y" position="0 2 0" scale="0.02 2 0.02" geometry="cube" uniforms="u_baseColorFactor:0 1 0 1"></xr-mesh>
    <xr-mesh node-id="mesh-z" position="0 0 2" scale="0.02 0.02 2" geometry="cube" uniforms="u_baseColorFactor:0 0 1 1"></xr-mesh>
    
    <xr-mesh 
      position="2 2 2" 
      rotation="90 0 0" 
      scale="1 0 1" 
      geometry="plane" 
      material="video-tsbs-mat" 
      uniforms="u_baseColorMap: video-test"
    ></xr-mesh>
  </xr-node>
  <xr-camera
      id="camera" node-id="camera" clear-color="0.925 0.925 0.925 1"
      background="ar" is-ar-camera
    ></xr-camera>
  <xr-node node-id="lights">
    <xr-light type="ambient" color="1 1 1" intensity="1" />
    <xr-light type="directional" rotation="180 0 0" color="1 1 1" intensity="3" />
  </xr-node>
</xr-scene>

Pages/index.js

var sceneReadyBehavior = require('../../behavior-scene/scene-ready');
var handleDecodedXML = require('../../behavior-scene/util').handleDecodedXML;

Page({
  behaviors:[sceneReadyBehavior],
  data: {
    renderWidth: 0,
    renderHeight: 0,
    width: 0,
    height: 0,
    top: 0,
    left: 0
  },
  
  onLoad: function() {
    const info = wx.getWindowInfo();
    
    
    this.setData({
      renderWidth: info.screenWidth,
      renderHeight: info.screenHeight,
      width: info.screenWidth,
      height: info.screenHeight,
      top: 0,
      left: 0
    });
  },

  handleSceneReady: function(e) {
    console.log('handleSceneReady called');
    this.scene = e.detail.value;
  },

  handleConfirmPosition: function() {
    console.log('handleConfirmPosition called');
    const arComponent = this.selectComponent('#main-frame');
    console.log('arComponent:', arComponent);
    if (arComponent) {
      arComponent.showModel();
    }
  }
});

Pages/index.wxml

<view class="container">
  <xr-ar-threeDof
    disable-scroll
    id="main-frame"
    width="{{renderWidth}}"
    height="{{renderHeight}}"
    style="width:{{width}}px;height:{{height}}px;top:{{top}}px;left:{{left}}px;display:block;"
    bind:sceneReady="handleSceneReady"
  /> 
  <view class="confirm-btn" bindtap="handleConfirmPosition">确认位置</view>
</view>