unity图像处理

图像处理主要有这些类

• Graphics

  • Blit 纹理通过材质渲染到目标，纹理设为材质的 _MainTex 属性
    将 dest 设置为渲染目标，为null时直接渲染到屏幕，使用后RenderTexture.active变成dest
    该操作在GPU上复制纹理，速度很快
    在 android 上如果不生效，尝试创建 RenderTexture 时深度值设为0
    scale 和 offset 不使用材质时才有，用来设置uv偏移和缩放
  • ConvertTexture 在不同格式和尺寸的纹理之间进行转换的有效方式
    注意 Blit 目标是 RenderTexture ，ConvertTexture 目标是 Texture2D
    ConvertTexture 对目标格式也有要求，必须是 RenderTarget 支持的格式，目前只测出支持 ARBG32
  • CopyTexture 高效复制，大小必须相同，格式必须一致
  • DrawMesh 绘制网格，只是提交到渲染队列
  • DrawTexture 在屏幕坐标系中绘制纹理,只能在OnGUI以及之后的生命周期方法中调用，否则无法显示。
  • SetRenderTarget 设置当前渲染目标

• Texture2D

  • ReadPixels 读取屏幕缓存到Texture2D，先通过 RenderTexture.active=XXX; 关联，就可以读取XXX
    该函数从GPU上读取像素到CPU
    如果只是用来获取像素数据，不需要渲染，则不要调用 Texture2D.Apply ，因为很耗时
    如果要用来渲染，则要调用 Texture2D.Apply
  • EncodeToPNG 编码成图像文件格式
  • EncodeToJPG

• ImageConversion 图像编解码

  • EncodeToPNG
  • EncodeToJPG
  • UnsafeEncodeNativeArrayToPNG
  • UnsafeEncodeNativeArrayToJPG

• RenderTexture 只包含GPU上的数据，要获取cpu上的数据（比如像素字节数组），需要使用 Texture2D.ReadPixels 从 GPU 传到 CPU

• ScreenCapture 截屏

  • CaptureScreenshot 截屏保存成png
  • CaptureScreenshotAsTexture 截屏保存成 Texture2D
  • CaptureScreenshotIntoRenderTexture 截屏保存成 RenderTexture

• WebCamTexture 捕获手机相机画面，可以参考 OpenCVForUnity\org\opencv\unity\helper\WebCamTextureToMatHelper.cs

图像处理的一般流程

• 渲染到 RenderTexture
  使用 Graphics.Blit
  或者 设置 Camera.targetTexture 然后调用 Camera.Render()
• 用 RenderTexture 生成 Texture2D
  使用 Texture2D.ReadPixels
• 将 Texture2D 保存成文件或设置给 RawImage 进行显示
  使用 Texture2D.EncodeToPNG

动态创建材质

//	shaderName 是可以在任何材质的着色器弹出窗口中看到的名称，例如“Standard”、“Unlit/Texture”、“Legacy Shaders/Diffuse”等
//	场景中必须有对象引用了着色器，否则着色器有可能缺失
//	可以直接把着色器添加到 ProjectSettings->Graphics->Built-in Shader Settings->Always included shaders
//	如果只是某个平台用，可以加到 ProjectSettings->Player->Android->Other Settings->PreloadAssets
Shader shader = Shader.Find(shaderName);	
Material material = new Material(shader);

创建RenderTexture

	public static RenderTexture CreateRenderTexture(int width, int height, int depth = 24, 
		RenderTextureFormat format = RenderTextureFormat.ARGB32, bool usequaAnti = true)
	{
		var rt = new RenderTexture(width, height, depth, format);
		rt.wrapMode = TextureWrapMode.Clamp;
		if (QualitySettings.antiAliasing > 0 && usequaAnti)
		{
			rt.antiAliasing = QualitySettings.antiAliasing;
		}
		rt.Create();
		return rt;
	}

	RenderTexture rt = CreateRenderTexture(1024,720,24,RenderTextureFormat.ARGB32);
	//	如果是临时使用，经常使用
	RenderTexture renderTexture = RenderTexture.GetTemporary(texture2D.width, texture2D.height);
	//	使用 renderTexture 后释放
	RenderTexture.ReleaseTemporary(renderTexture);

RenderTexture 和 Texture2D 互转

RenderTexture 代表GPU纹理，Texture2D 代表CPU纹理，当要操作纹理时使用 RenderTexture 更快，当要操作数据时使用 Texture2D

• RenderTexture 转 Texture2D

  1. 方法一:

     	Texture2D tex = new Texture2D(renderTexture.width, renderTexture.height, TextureFormat.RGB24, false);
     	var prevActive = RenderTexture.active;
     	RenderTexture.active = renderTexture;
     	
     	// ReadPixels 读取当前渲染目标某个区域的像素并写入 tex 中，[0,0] 表示左下
     	tex.ReadPixels(new Rect(0, 0, renderTexture.width, renderTexture.height), 0, 0);
     	//	Apply 把纹理数据上传给GPU，如果这个纹理只是用来获取像素数据，不需要渲染，则不要调用该函数，因为很耗时
     	tex.Apply();
     	
     	RenderTexture.active = prevActive;
     

  2. 方法二:

     	renderTexture.enableRandomWrite = true;
     	Texture2D tex = new Texture2D(renderTexture.width, renderTexture.height, TextureFormat.RGB24, false);
     	Graphics.ConvertTexture(renderTexture, tex);  //RT要enableRandomWrite，T2D要关闭mipmap，否则报错误
     

• Texture2D 转 RenderTexture

  	//	如果只是临时使用，用这个效率更高
  	RenderTexture renderTexture = RenderTexture.GetTemporary(texture2D.width, texture2D.height);
  	RenderTexture prev = RenderTexture.active;
  	RenderTexture.active = renderTexture;
  	Graphics.Blit(texture2D, renderTexture);
  	RenderTexture.active = prev;
  	RenderTexture.ReleaseTemporary(renderTexture);
  

获取 RenderTexture 图像数据

```csharp
//	RenderTexture 数据是在 GPU 上，要获取数据必须先传到 cpu 上
byte[] pixelBuffer = new byte[renderTexture.width * renderTexture.height * 4];
if (SystemInfo.supportsAsyncGPUReadback)
	AsyncGPUReadback.Request(renderTexture, 0, request => {
		if (pixelBuffer != null) {
			request.GetData<byte>().CopyTo(pixelBuffer);
		}
	});
else {
	var texture2D = new Texture2D(renderTexture.width, renderTexture.height, TextureFormat.RGBA32, false, false);
	var prevActive = RenderTexture.active;
	RenderTexture.active = renderTexture;
	texture2D.ReadPixels(new Rect(0, 0, renderTexture.width, renderTexture.height), 0, 0, false);
	texture2D.GetRawTextureData<byte>().CopyTo(pixelBuffer);
	RenderTexture.active = prevActive;
}
```

纹理和图像互转

主要使用 UnityEngine.ImageConversion

• 纹理编码成文件格式
  byte[] data = texture2D.EncodeToTGA();
  byte[] data = texture2D.EncodeToJPG();
  byte[] data = texture2D.EncodeToPNG();
  下面这句跟上面那句等价
  byte[] bytes = ImageConversion.EncodeArrayToPNG(texture2D.GetRawTextureData(), texture2D.graphicsFormat, (uint)texture2D.width, (uint)texture2D.height);

• 加载图片成纹理

  	FileStream fs = new System.IO.FileStream("d:\\1.jpg", System.IO.FileMode.Open, System.IO.FileAccess.Read);
  	byte[] thebytes = new byte[fs.Length];
  	fs.Read(thebytes, 0, (int)fs.Length);
  
  	//实例化一个Texture2D，宽和高设置可以是任意的，因为当使用LoadImage方法会对Texture2D的宽和高会做相应的调整
  	Texture2D texture = new Texture2D(1,1);
  	texture.LoadImage(thebytes);
  
  	material.mainTexture = texture;
  	material.mainTextureScale = new Vector2(1, 1);
  	material.mainTextureOffset = new Vector2(0, 0);
  

• 直接设置纹理数据

  	Texture2D tex = new Texture2D(16, 16, TextureFormat.R8, false);
  	//	bytes 长度为 w * h * 每像素占的字节数，如果有 mipmap，还要加上 mipmap 大小
  	byte[] bytes = new byte[tex.width*tex.height];	
  	for (int y = 0; y < tex.height; y++)
      {
          for (int x = 0; x < tex.width; x++)
          {
              bytes[index++] = 255;
          }
      }
  	//	如果要解压缩数据，可以直接修改 tex.GetRawTextureData() 返回的数组，省去1次拷贝
  	tex.LoadRawTextureData(bytes);
      tex.Apply();
  

• UI中显示纹理

  	//	使用 RawImage
  	rawImage.texture = texture2D;
  	//	使用 Image，必须先创建 Sprite
  	Sprite sprite = Sprite.Create(texture2D, new Rect(0, 0, texture2D.width, texture2D.height), new Vector2(0, 0));
  	image.sprite = sprite;
  

• 截图

  	Texture2D tex = new Texture2D(Screen.width, Screen.height, TextureFormat.RGB24, false);
      // ReadPixels 读取当前渲染目标某个区域的像素并写入 tex 中
      tex.ReadPixels(new Rect(0, 0, width, height), 0, 0);
      tex.Apply();
  	byte[] bytes = tex.EncodeToPNG();
  	File.WriteAllBytes(Application.dataPath + "/../SavedScreen.png", bytes);
  

Texture2D 保存成文件

	public void SaveTextureToFile(Texture2D texture, string filePath)
	{
		var bytes = texture.EncodeToPNG();
		var file = File.Open(filePath, FileMode.Create);
		var binary = new BinaryWriter(file);
		binary.Write(bytes);
		file.Close();
	}

使用 ReadPixels 拼接图像

	RenderTexture prev = RenderTexture.active;
	RenderTexture.active = m_RGBSource;
	tempdest.ReadPixels(new Rect(0, 0, m_RGBSource.width, m_RGBSource.height), 0, 0);

	RenderTexture.active = foresource;
	tempdest.ReadPixels(new Rect(0, 0, foresource.width, foresource.height), foresource.width, 0);
	tempdest.Apply();
	RenderTexture.active = prev;

	Graphics.Blit(tempdest, dest);

相机渲染到纹理

	var main = Camera.main;
	var renderTexture = new RenderTexture(Screen.width, Screen.height, 16);
	main.targetTexture = renderTexture;
	main.Render();
	RenderTexture.active = renderTexture;
	Texture2D texture = new Texture2D(Screen.width, Screen.height, TextureFormat.RGBA32, false, false);
	texture.ReadPixels(new Rect(0, 0, Screen.width, Screen.height), 0, 0, false);

对纹理进行shader处理

	//	这个 shader 我们一般创建时选择  ImageEffectShader
	Material material = new Material(Resources.Load<Shader>(shader_name));
	RenderTexture renderTexture = RenderTexture.GetTemporary(texture2D.width, texture2D.height);
	//	Blit 将 dest 设置为渲染目标，在材质上设置 source _MainTex 属性， 并绘制全屏四边形
	Graphics.Blit(texture2D, renderTexture, material);
	RenderTexture.ReleaseTemporary(renderTexture);

图像缩放

• 使用 Graphics.Blit/ConvertTexture 缩放(推荐)
  如果只是缩放用 ConvertTexture，如果涉及颜色变换等就用 Blit

  	//	用 ConvertTexture 缩放
  	static Texture2D Resize(Texture2D source, int newWidth, int newHeight)
  	{
  		Texture2D tex = new Texture2D(newWidth, newHeight,TextureFormat.ARGB32,false);
  		Graphics.ConvertTexture(source, tex);
  		return tex;
  	}
  	
  	//	用 Blit 缩放
  	static Texture2D Resize(Texture2D source, int newWidth, int newHeight)
  	{
  		source.filterMode = FilterMode.Point;
  		RenderTexture rt = RenderTexture.GetTemporary(newWidth, newHeight);
  		rt.filterMode = FilterMode.Point;
  		RenderTexture prev = RenderTexture.active;
  		RenderTexture.active = rt;
  		Graphics.Blit(source, rt);
  		var nTex = new Texture2D(newWidth, newHeight);
  		nTex.ReadPixels(new Rect(0, 0, newWidth, newHeight), 0, 0);
  		nTex.Apply();
  		RenderTexture.active = prev;
  		RenderTexture.ReleaseTemporary(renderTexture);
  		return nTex;
  	}
  

• 直接缩放(耗性能)

  	Texture2D ScaleTexture(Texture2D source, int targetWidth, int targetHeight)
  	{
  		Texture2D result = new Texture2D(targetWidth, targetHeight, source.format, false);
  
  		float incX = (1.0f / (float)targetWidth);
  		float incY = (1.0f / (float)targetHeight);
  
  		for (int i = 0; i < result.height; ++i)
  		{
  			for (int j = 0; j < result.width; ++j)
  			{
  				Color newColor = source.GetPixelBilinear((float)j / (float)result.width, (float)i / (float)result.height);
  				result.SetPixel(j, i, newColor);
  			}
  		}
  
  		result.Apply();
  		return result;
  	}
  

• 使用gpu计算(略显复杂)

  • c#代码:

    	RenderTexture Resize(ComputeShader shader, int divideSize) {
    		RenderTexture t = new RenderTexture(inputTexture.width/ divideSize, inputTexture.height / divideSize, 24, RenderTextureFormat.ARGBFloat);
    		t.enableRandomWrite = true;
    		t.Create();
    		int k = shader.FindKernel("Resize");
    		shader.SetInt("divideSize", divideSize);        
    		shader.SetTexture(k, "inputTexture", inputTexture);
    		shader.SetTexture(k, "outputTexture", t);
    		shader.Dispatch(k, inputTexture.width / 8, inputTexture.height / 8, 1);
    		return t;
    	}
    

  • Compute Shader 代码:

    	#pragma kernel Resize
    
    	int divideSize; // 原始圖像大小除以這個值
    	Texture2D inputTexture;
    	RWTexture2D <float4> outputTexture;
    
    	[numthreads(8, 8, 1)]
    	void Resize(uint3 id : SV_DispatchThreadID) 
    	{
    		outputTexture[id.xy / divideSize] = inputTexture[id.xy];
    	}