Skip to content

Instantly share code, notes, and snippets.

@Thaina

Thaina/CameraController.cs

Last active December 4, 2023 03:24
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save Thaina/c4cea4f108da718924558fb2f7ff66df to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save Thaina/c4cea4f108da718924558fb2f7ff66df to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. Thaina renamed this gist Dec 4, 2023. 1 changed file with 0 additions and 0 deletions.
    0 gistfile1.txt → CameraController.cs
    View file
    File renamed without changes.
  2. Thaina revised this gist Dec 4, 2023. 2 changed files with 231 additions and 3 deletions.
    163 changes: 160 additions & 3 deletions RelativeOffcenter.cs
    View file
    Original file line number Diff line number Diff line change
    @@ -1,8 +1,16 @@
    using System;
    using System.Linq;
    using System.Collections.Generic;
    using System.Runtime.InteropServices;

    using UnityEngine;
    using TMPro;
    using Newtonsoft.Json.Linq;
    using Newtonsoft.Json;

    #if UNITY_ANDROID
    using UnityEngine.Android;
    #endif

    [ExecuteInEditMode]
    public class RelativeOffcenter : MonoBehaviour
    @@ -14,12 +22,47 @@ public class RelativeOffcenter : MonoBehaviour
    [RuntimeInitializeOnLoadMethod]
    static void Init()
    {
    JsonConvert.DefaultSettings = () => {
    var serializer = new JsonSerializerSettings();
    serializer.Converters.Add(new VectorConverter());
    return serializer;
    };

    #if UNITY_EDITOR_WIN || UNITY_STANDALONE_WIN
    var hwnd = GetDesktopWindow();
    var hdc = GetDC(hwnd);
    monitorSizeM.x = GetDeviceCaps(hdc,HORZSIZE) / 1000f;
    monitorSizeM.y = GetDeviceCaps(hdc,VERTSIZE) / 1000f;
    ReleaseDC(hwnd,hdc);

    cameraCharacteristics = new [] {
    new CameraCharacteristics() {
    FocalLengths = new float[] { 2.4f },
    SensorSizeMM = new Vector2(3.2128f,2.4128f),
    SensorSizePx = new Vector2Int(4016,3016),
    }
    };
    #elif UNITY_ANDROID
    var unityPlayer = new AndroidJavaClass("com.unity3d.player.UnityPlayer");
    var activity = unityPlayer.GetStatic<AndroidJavaObject>("currentActivity");
    var context = activity.Call<AndroidJavaObject>("getApplicationContext");

    var displayMetrics = activity.Call<AndroidJavaObject>("getResources").Call<AndroidJavaObject>("getDisplayMetrics");

    var dpi = new Vector2(displayMetrics.Get<float>("xdpi"),displayMetrics.Get<float>("ydpi"));
    var pixels = new Vector2(displayMetrics.Get<int>("widthPixels"),displayMetrics.Get<int>("heightPixels"));

    monitorSizeM = pixels * 25.4f / (dpi * 1000);

    var cameraManager = context.Call<AndroidJavaObject>("getSystemService",context.GetStatic<string>("CAMERA_SERVICE"));
    Debug.LogFormat("cameraManager : {0}",cameraManager);

    var cameraIDs = cameraManager.Call<string[]>("getCameraIdList");
    Debug.LogFormat("cameraIDs : {0}",string.Join(" | ",cameraIDs));

    cameraCharacteristics = cameraIDs.Select((cameraID) => CameraCharacteristics.FromCameraID(cameraManager,cameraID)).ToArray();
    #else
    monitorSizeM = new Vector2(Screen.currentResolution.width,Screen.currentResolution.height) * 25.4f / (Screen.dpi * 1000);
    #endif
    }

    @@ -36,12 +79,124 @@ static void Init()
    static extern int ReleaseDC(IntPtr hwnd,IntPtr hdc);
    #endif

    public enum PreferFacing { Front = 0,Back = 1,None = 2 }
    public struct CameraCharacteristics
    {
    public string CamerID;
    public PreferFacing preferFacing;
    public Vector2Int SensorSizePx;
    public Vector2 SensorSizeMM;
    public Vector3 LensPosition;
    public Quaternion LensRotation;
    public float[] FocalLengths;

    public Vector2 PixelToMM(Vector2 px) => px * SensorSizeMM / SensorSizePx;

    public Vector2 DistanceMM(int findex,Vector2 px,Vector2 realSize) => DistanceMM(FocalLengths[findex],PixelToMM(px),realSize);

    public static float DistanceMM(in float focalLength,in float lensSizeMM,in float realSizeMM)
    {
    return focalLength + (focalLength * realSizeMM / lensSizeMM);
    }

    public static Vector2 DistanceMM(in float focalLength,in Vector2 lensSizeMM,in Vector2 realSizeMM)
    {
    return new Vector2(DistanceMM(focalLength,lensSizeMM.x,realSizeMM.x),DistanceMM(focalLength,lensSizeMM.y,realSizeMM.y));
    }

    #if UNITY_ANDROID
    public static CameraCharacteristics FromCameraID(AndroidJavaObject cameraManager,string cameraID)
    {
    var CameraCharacteristics = cameraManager.Call<AndroidJavaObject>("getCameraCharacteristics",cameraID);

    CameraCharacteristics data;

    data.CamerID = cameraID;
    data.preferFacing = (PreferFacing)CameraCharacteristics.GetFromConstantName<AndroidJavaObject>("LENS_FACING").Call<int>("intValue");
    data.FocalLengths = CameraCharacteristics.GetFromConstantName<float[]>("LENS_INFO_AVAILABLE_FOCAL_LENGTHS");
    data.SensorSizeMM = CameraCharacteristics.GetFromConstantName<AndroidJavaObject>("SENSOR_INFO_PHYSICAL_SIZE").ConvertFromSizeF();
    data.SensorSizePx = CameraCharacteristics.GetFromConstantName<AndroidJavaObject>("SENSOR_INFO_PIXEL_ARRAY_SIZE").ConvertFromSize();
    data.LensPosition = CameraCharacteristics.GetFromConstantName<float[]>("LENS_POSE_TRANSLATION").ReadVector3();
    data.LensRotation = CameraCharacteristics.GetFromConstantName<float[]>("LENS_POSE_ROTATION").ReadQuaternion();

    return data;
    }
    #endif
    }

    public TMP_Text text;
    public static CameraCharacteristics[] cameraCharacteristics;
    void Start()
    {
    Init();
    text.text = JArray.FromObject(cameraCharacteristics).ToString(Formatting.Indented);
    }

    Rect rect;
    public class VectorConverter : JsonConverter
    {
    public override bool CanConvert(Type objectType)
    {
    return objectType == typeof(Vector2) || objectType == typeof(Vector3) || objectType == typeof(Vector4) || objectType == typeof(Quaternion);
    }

    public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
    {
    var t = serializer.Deserialize(reader);
    return JsonConvert.DeserializeObject(t.ToString(),objectType);
    }

    public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
    {
    if(value is Vector2 v2)
    {
    writer.WriteStartObject();
    writer.WritePropertyName("x");
    writer.WriteValue(v2.x);
    writer.WritePropertyName("y");
    writer.WriteValue(v2.y);
    writer.WriteEndObject();
    }
    else if(value is Vector3 v3)
    {
    writer.WriteStartObject();
    writer.WritePropertyName("x");
    writer.WriteValue(v3.x);
    writer.WritePropertyName("y");
    writer.WriteValue(v3.y);
    writer.WritePropertyName("z");
    writer.WriteValue(v3.z);
    writer.WriteEndObject();
    }
    else if(value is Vector4 v4)
    {
    writer.WriteStartObject();
    writer.WritePropertyName("x");
    writer.WriteValue(v4.x);
    writer.WritePropertyName("y");
    writer.WriteValue(v4.y);
    writer.WritePropertyName("z");
    writer.WriteValue(v4.z);
    writer.WritePropertyName("w");
    writer.WriteValue(v4.w);
    writer.WriteEndObject();
    }
    else if(value is Quaternion q)
    {
    writer.WriteStartObject();
    writer.WritePropertyName("x");
    writer.WriteValue(q.x);
    writer.WritePropertyName("y");
    writer.WriteValue(q.y);
    writer.WritePropertyName("z");
    writer.WriteValue(q.z);
    writer.WritePropertyName("w");
    writer.WriteValue(q.w);
    writer.WriteEndObject();
    }
    }
    }


    Rect rect;
    Vector3 center;
    new Camera camera;
    void LateUpdate()
    @@ -71,10 +226,11 @@ void LateUpdate()
    rect.max = rect.min + (monitorRealScale * new Vector2(Screen.width,Screen.height));

    var frame = rect;
    frame.position += (Vector2)gameObject.transform.localPosition;
    frame.position -= (Vector2)gameObject.transform.localPosition;
    camera.projectionMatrix = Unity.Mathematics.float4x4.PerspectiveOffCenter(frame.xMin,frame.xMax,-frame.yMax,-frame.yMin,camera.nearClipPlane,camera.farClipPlane);
    }

    #if UNITY_EDITOR
    void OnDrawGizmos()
    {
    if(!camera)
    @@ -121,4 +277,5 @@ static IEnumerable<Vector3> TransformRectPointFromMinMax(Transform transform,Rec
    yield return (prev,first);
    }
    }
    #endif
    }
    71 changes: 71 additions & 0 deletions gistfile1.txt
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,71 @@
    using System.Collections;
    using System.Collections.Generic;

    using UnityEngine;
    using UnityEngine.Android;

    using Mediapipe.Unity;
    using Mediapipe.Unity.IrisTracking;

    using TMPro;
    using System.Linq;

    public class CameraController : MonoBehaviour
    {
    public IrisTrackingGraph irisTrackingGraph;
    public Transform camera;
    public TMP_Text text;
    public Vector2 leftEyeSize,rightEyeSize;
    IEnumerator Start()
    {
    Debug.Log("Start");

    Permission.RequestUserPermission(Permission.Camera);

    while(!(ImageSourceProvider.ImageSource is var source && source && source.isPlaying))
    yield return new WaitForEndOfFrame();

    Debug.LogFormat("ImageSourceProvider.ImageSource : {0}",ImageSourceProvider.ImageSource);

    while(irisTrackingGraph._faceLandmarksWithIrisStream == null)
    yield return new WaitForEndOfFrame();

    Debug.LogFormat("irisTrackingGraph : {0}",irisTrackingGraph._faceLandmarksWithIrisStream);
    irisTrackingGraph._faceLandmarksWithIrisStream.AddListener((sender,output) => {
    if(output?.value?.Landmark == null)
    {
    Debug.LogError("No landmark detected");
    return;
    }

    var parts = FaceLandmarkListWithIrisAnnotation.PartitionLandmarkList(output.value.Landmark);
    if(parts.leftIris == null && parts.rightIris == null)
    {
    Debug.LogError("No iris detected");
    return;
    }

    leftEyeSize = new Vector2(parts.leftIris.Distance(1,3).Value,parts.leftIris.Distance(2,4).Value);
    rightEyeSize = new Vector2(parts.rightIris.Distance(1,3).Value,parts.rightIris.Distance(2,4).Value);
    });
    }

    WebCamSource webCamSource;
    void Update()
    {
    if(!webCamSource)
    webCamSource = GameObject.FindFirstObjectByType<WebCamSource>();

    if(text && RelativeOffcenter.cameraCharacteristics?.Length > 0 && webCamSource?.webCamDevice?.isFrontFacing is bool facing)
    {
    var preferFacing = facing ? RelativeOffcenter.PreferFacing.Front : RelativeOffcenter.PreferFacing.Back;

    var cc = RelativeOffcenter.cameraCharacteristics.OrderBy((characteristics) => characteristics.preferFacing == preferFacing ? 0 : 1).First();

    var eyeSize = 11.75f * Vector2.one;
    var dl = cc.DistanceMM(0,leftEyeSize * cc.SensorSizePx,eyeSize) / 1000;
    var dr = cc.DistanceMM(0,rightEyeSize * cc.SensorSizePx,eyeSize) / 1000;
    text.text = string.Join("\n", "L : " + leftEyeSize.ToString("0.0000"),"R : " + rightEyeSize.ToString("0.0000"),"DL : " + dl.ToString("0.0000"),"DR : " + dr.ToString("0.0000"));
    }
    }
    }
  3. Thaina created this gist Nov 26, 2023.
    124 changes: 124 additions & 0 deletions RelativeOffcenter.cs
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,124 @@
    using System;
    using System.Collections.Generic;
    using System.Runtime.InteropServices;

    using UnityEngine;

    [ExecuteInEditMode]
    public class RelativeOffcenter : MonoBehaviour
    {
    public static Vector2 monitorSizeM;
    #if UNITY_EDITOR
    [UnityEditor.InitializeOnLoadMethod]
    #endif
    [RuntimeInitializeOnLoadMethod]
    static void Init()
    {
    #if UNITY_EDITOR_WIN || UNITY_STANDALONE_WIN
    var hwnd = GetDesktopWindow();
    var hdc = GetDC(hwnd);
    monitorSizeM.x = GetDeviceCaps(hdc,HORZSIZE) / 1000f;
    monitorSizeM.y = GetDeviceCaps(hdc,VERTSIZE) / 1000f;
    ReleaseDC(hwnd,hdc);
    #endif
    }

    #if UNITY_EDITOR_WIN || UNITY_STANDALONE_WIN
    const int HORZSIZE = 4;
    const int VERTSIZE = 6;
    [DllImport("gdi32.dll")]
    static extern int GetDeviceCaps(IntPtr hdc, int nIndex);
    [DllImport("user32.dll", SetLastError = false)]
    static extern IntPtr GetDesktopWindow();
    [DllImport("user32.dll", SetLastError = false)]
    static extern IntPtr GetDC(IntPtr hwnd);
    [DllImport("user32.dll", SetLastError = false)]
    static extern int ReleaseDC(IntPtr hwnd,IntPtr hdc);
    #endif

    void Start()
    {
    Init();
    }

    Rect rect;
    Vector3 center;
    new Camera camera;
    void LateUpdate()
    {
    if(!camera)
    camera = gameObject.GetComponent<Camera>();

    var focus = gameObject.transform.parent ? gameObject.transform.parent.position : Vector3.zero;

    var camLoc = camera.gameObject.transform;
    var ray = new Ray(camLoc.position,camLoc.forward);

    camera.nearClipPlane = Vector3.Dot(ray.direction,focus - ray.origin);
    center = focus - (ray.direction * camera.nearClipPlane);

    var monitorSizePixels = new Vector2(Screen.currentResolution.width,Screen.currentResolution.height);
    var monitorRealScale = monitorSizeM / monitorSizePixels;
    var monitorCenter = monitorSizePixels / 2;
    #if UNITY_EDITOR
    if(Screen.mainWindowPosition.sqrMagnitude > 0)
    rect = new Rect(Screen.mainWindowPosition,new Vector2(Screen.width,Screen.height));
    else rect = UnityEditor.SceneView.lastActiveSceneView.position;
    #else
    rect = new Rect(Screen.mainWindowPosition,new Vector2(Screen.width,Screen.height));
    #endif
    rect.min = monitorRealScale * (rect.min - monitorCenter);
    rect.max = rect.min + (monitorRealScale * new Vector2(Screen.width,Screen.height));

    var frame = rect;
    frame.position += (Vector2)gameObject.transform.localPosition;
    camera.projectionMatrix = Unity.Mathematics.float4x4.PerspectiveOffCenter(frame.xMin,frame.xMax,-frame.yMax,-frame.yMin,camera.nearClipPlane,camera.farClipPlane);
    }

    void OnDrawGizmos()
    {
    if(!camera)
    camera = gameObject.GetComponent<Camera>();

    var offset = center + (camera.gameObject.transform.forward * camera.nearClipPlane);
    Gizmos.color = Color.green;
    foreach(var (from,to) in PointLoopToLine(TransformRectPointFromMinMax(gameObject.transform,new Rect(-0.5f * monitorSizeM,monitorSizeM))))
    Gizmos.DrawLine(offset + from,offset + to);

    Gizmos.color = Color.red;
    foreach(var (from,to) in PointLoopToLine(TransformRectPointFromMinMax(gameObject.transform,rect)))
    Gizmos.DrawLine(offset + from,offset + to);

    static IEnumerable<Vector3> TransformRectPointFromMinMax(Transform transform,Rect rect)
    {
    var up = transform.up;
    var right = transform.right;
    yield return (right * rect.xMin) - (up * rect.yMin);
    yield return (right * rect.xMax) - (up * rect.yMin);
    yield return (right * rect.xMax) - (up * rect.yMax);
    yield return (right * rect.xMin) - (up * rect.yMax);
    }

    static IEnumerable<(Vector3,Vector3)> PointLoopToLine(IEnumerable<Vector3> points)
    {
    var itor = points.GetEnumerator();
    if(!itor.MoveNext())
    yield break;

    var first = itor.Current;
    if(!itor.MoveNext())
    yield break;

    var prev = first;
    do
    {
    var current = itor.Current;
    yield return (prev,current);
    prev = current;
    }
    while(itor.MoveNext());

    yield return (prev,first);
    }
    }
    }