Created
May 19, 2021 18:57
-
-
Save miff/fd1398bd57624b4175cbcbdb3d096951 to your computer and use it in GitHub Desktop.
Custom Composition example
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import AVFoundation | |
| import Foundation | |
| import UIKit | |
| class CustomComposition: NSObject, AVVideoCompositing { | |
| var sourcePixelBufferAttributes: [String : Any]? { | |
| get { | |
| return ["\(kCVPixelBufferPixelFormatTypeKey)": kCVPixelFormatType_32BGRA] | |
| } | |
| } | |
| var requiredPixelBufferAttributesForRenderContext: [String : Any] { | |
| get { | |
| return ["\(kCVPixelBufferPixelFormatTypeKey)": kCVPixelFormatType_32BGRA] | |
| } | |
| } | |
| var pass: Int = 0 | |
| let renderQueue = DispatchQueue(label: "\(Bundle.main.bundleIdentifier ?? "").renderingqueue", attributes: []) | |
| let renderContextQueue = DispatchQueue(label: "\(Bundle.main.bundleIdentifier ?? "").rendercontextqueue", attributes: []) | |
| func renderContextChanged(_ newRenderContext: AVVideoCompositionRenderContext) { } | |
| func startRequest(_ asyncVideoCompositionRequest: AVAsynchronousVideoCompositionRequest) { | |
| autoreleasepool() { | |
| self.renderQueue.sync { | |
| let request = asyncVideoCompositionRequest | |
| //Destination Buffer | |
| let destination = request.renderContext.newPixelBuffer() | |
| //Grab Frames | |
| if request.sourceTrackIDs.count != 2 { return } | |
| guard let front = request.sourceFrame(byTrackID: 2) else { return request.finish(with: NSError(domain: "", code: 1, userInfo: nil)) } | |
| guard let back = request.sourceFrame(byTrackID: 1) else { return request.finish(with: NSError(domain: "", code: 2, userInfo: nil)) } | |
| CVPixelBufferLockBaseAddress(front, CVPixelBufferLockFlags.readOnly) | |
| CVPixelBufferLockBaseAddress(back, CVPixelBufferLockFlags.readOnly) | |
| CVPixelBufferLockBaseAddress(destination!, CVPixelBufferLockFlags.readOnly) | |
| renderFromBuffer(destination: destination!, front: front, back: back) | |
| CVPixelBufferUnlockBaseAddress(destination!, CVPixelBufferLockFlags.readOnly) | |
| CVPixelBufferUnlockBaseAddress(back, CVPixelBufferLockFlags.readOnly) | |
| CVPixelBufferUnlockBaseAddress(front, CVPixelBufferLockFlags.readOnly) | |
| request.finish(withComposedVideoFrame: destination!) | |
| DispatchQueue.main.async { | |
| NotificationCenter.default.post(name: Notification.Name("frame.render.pass"), object: nil, userInfo: ["pass": self.pass]) | |
| } | |
| } | |
| } | |
| } | |
| private func renderFromBuffer(destination: CVPixelBuffer, front: CVPixelBuffer, back: CVPixelBuffer) { | |
| let width = CVPixelBufferGetWidth(destination) | |
| let height = CVPixelBufferGetHeight(destination) | |
| let newContext = CGContext(data: CVPixelBufferGetBaseAddress(destination), width: width, height: height, bitsPerComponent: 8, bytesPerRow: CVPixelBufferGetBytesPerRow(destination), space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) | |
| let frontImage = createSourceImageFromReferance(buffer: front, isFront: true) | |
| let backImage = createSourceImageFromReferance(buffer: back) | |
| //DRAW 'BACK' IMAGE | |
| newContext?.saveGState() | |
| newContext?.translateBy(x: 0, y: CGFloat(height)) | |
| newContext?.scaleBy(x: 1.0, y: -1.0) | |
| let backImagerect = CGRect(x: 0, y: 0, width: width / 2, height: height) | |
| newContext?.draw(backImage, in: backImagerect) | |
| newContext?.scaleBy(x: 1.0, y: -1.0) | |
| newContext?.translateBy(x: 0, y: -CGFloat(height / 2)) | |
| newContext?.restoreGState() | |
| //DRAW 'FRONT' IMAGE AT CENTER | |
| newContext?.saveGState() | |
| newContext?.translateBy(x: 0, y: CGFloat(height)) | |
| newContext?.scaleBy(x: 1.0, y: -1.0) | |
| let w: CGFloat = 506//CGFloat(frontImage.width) * 0.564 | |
| let h: CGFloat = 720//CGFloat(frontImage.height) * 0.564 | |
| let frontImageSize = CGSize(width: w , height: h) | |
| let centerPoint = CGPoint(x: 406, y: 0) | |
| let frontImagerect = CGRect(origin: centerPoint, size: frontImageSize) | |
| newContext?.draw(frontImage, in: frontImagerect) | |
| newContext?.scaleBy(x: 1.0, y: -1.0) | |
| newContext?.translateBy(x: 0, y: -CGFloat(height / 2)) | |
| newContext?.restoreGState() | |
| pass += 1 | |
| newContext?.flush() | |
| } | |
| private func createSourceImageFromReferance(buffer: CVPixelBuffer, isFront: Bool = false) -> CGImage { | |
| let width = CVPixelBufferGetWidth(buffer) | |
| let height = CVPixelBufferGetHeight(buffer) | |
| let stride = CVPixelBufferGetBytesPerRow(buffer) | |
| let data = CVPixelBufferGetBaseAddress(buffer) | |
| let rgb = CGColorSpaceCreateDeviceRGB() | |
| let releaseMaskImagePixelData: CGDataProviderReleaseDataCallback = { (info: UnsafeMutableRawPointer?, data: UnsafeRawPointer, size: Int) -> () in | |
| // https://developer.apple.com/reference/coregraphics/cgdataproviderreleasedatacallback | |
| return | |
| } | |
| let provider: CGDataProvider? = CGDataProvider(dataInfo: nil, data: data!, size: height * stride, releaseData: releaseMaskImagePixelData) | |
| let last = CGBitmapInfo(rawValue: CGBitmapInfo.byteOrder32Big.rawValue | CGImageAlphaInfo.premultipliedLast.rawValue) | |
| let image = CGImage(width: width, height: height, bitsPerComponent: 8, bitsPerPixel: 32, bytesPerRow: stride, space: rgb, bitmapInfo: last, provider: provider!, decode: nil, shouldInterpolate: false, intent: .defaultIntent) | |
| let rect = CGRect(x: 0, y: 0, width: Int(width), height: Int(height)) | |
| var newContext = CGContext(data: nil, width: width, height: height, bitsPerComponent: 8, bytesPerRow: CVPixelBufferGetBytesPerRow(buffer), space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) | |
| if isFront { | |
| newContext = CGContext(data: nil, width: height, height: width, bitsPerComponent: 8, bytesPerRow: CVPixelBufferGetBytesPerRow(buffer), space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) | |
| } | |
| newContext?.saveGState() | |
| newContext?.setFillColor(UIColor.red.cgColor) | |
| newContext?.fillPath() | |
| if isFront { | |
| newContext?.translateBy(x: 1, y: -1) | |
| newContext?.rotate(by: .pi / 2) | |
| } else { | |
| newContext?.translateBy(x: 0, y: CGFloat(height)) | |
| } | |
| newContext?.scaleBy(x: 1, y: -1) | |
| newContext?.draw(image!, in: rect) | |
| newContext?.restoreGState() | |
| let im = (newContext!.makeImage())! | |
| newContext?.flush() | |
| return im | |
| } | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment