[{"data":1,"prerenderedAt":3258},["ShallowReactive",2],{"i-ph:moon-bold":3,"i-ph:globe-simple":8,"i-ph:caret-down":10,"i-ph:list":12,"i-ph:heart-fill":14,"i-ph:discord-logo-bold":16,"i-ph:github-logo-bold":18,"i-ph:cookie-bold":20,"blog-article-en-screen-mirror":22,"i-ph:arrow-left":3256},{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":7},0,256,false,"\u003Cpath fill=\"currentColor\" d=\"M236.37 139.4a12 12 0 0 0-12-3A84.07 84.07 0 0 1 119.6 31.59a12 12 0 0 0-15-15a108.86 108.86 0 0 0-54.91 38.48A108 108 0 0 0 136 228a107.1 107.1 0 0 0 64.93-21.69a108.86 108.86 0 0 0 38.44-54.94a12 12 0 0 0-3-11.97m-49.88 47.74A84 84 0 0 1 68.86 69.51a84.9 84.9 0 0 1 23.41-21.22Q92 52.13 92 56a108.12 108.12 0 0 0 108 108q3.87 0 7.71-.27a84.8 84.8 0 0 1-21.22 23.41\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":9},"\u003Cpath fill=\"currentColor\" d=\"M128 24a104 104 0 1 0 104 104A104.12 104.12 0 0 0 128 24m87.62 96h-39.83c-1.79-36.51-15.85-62.33-27.38-77.6a88.19 88.19 0 0 1 67.22 77.6ZM96.23 136h63.54c-2.31 41.61-22.23 67.11-31.77 77c-9.55-9.9-29.46-35.4-31.77-77m0-16c2.31-41.61 22.23-67.11 31.77-77c9.55 9.93 29.46 35.43 31.77 77Zm11.36-77.6C96.06 57.67 82 83.49 80.21 120H40.37a88.19 88.19 0 0 1 67.22-77.6M40.37 136h39.84c1.82 36.51 15.85 62.33 27.38 77.6A88.19 88.19 0 0 1 40.37 136m108 77.6c11.53-15.27 25.56-41.09 27.38-77.6h39.84a88.19 88.19 0 0 1-67.18 77.6Z\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":11},"\u003Cpath fill=\"currentColor\" d=\"m213.66 101.66l-80 80a8 8 0 0 1-11.32 0l-80-80a8 8 0 0 1 11.32-11.32L128 164.69l74.34-74.35a8 8 0 0 1 11.32 11.32\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":13},"\u003Cpath fill=\"currentColor\" d=\"M224 128a8 8 0 0 1-8 8H40a8 8 0 0 1 0-16h176a8 8 0 0 1 8 8M40 72h176a8 8 0 0 0 0-16H40a8 8 0 0 0 0 16m176 112H40a8 8 0 0 0 0 16h176a8 8 0 0 0 0-16\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":15},"\u003Cpath fill=\"currentColor\" d=\"M240 102c0 70-103.79 126.66-108.21 129a8 8 0 0 1-7.58 0C119.79 228.66 16 172 16 102a62.07 62.07 0 0 1 62-62c20.65 0 38.73 8.88 50 23.89C139.27 48.88 157.35 40 178 40a62.07 62.07 0 0 1 62 62\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":17},"\u003Cpath fill=\"currentColor\" d=\"M108 136a16 16 0 1 1-16-16a16 16 0 0 1 16 16m56-16a16 16 0 1 0 16 16a16 16 0 0 0-16-16m76.07 76.56l-67 29.71A20.15 20.15 0 0 1 146 214.9l-8.54-23.13c-3.13.14-6.27.24-9.45.24s-6.32-.1-9.45-.24L110 214.9a20.19 20.19 0 0 1-27.08 11.37l-67-29.71a19.93 19.93 0 0 1-11.3-23.15L34.15 57a20 20 0 0 1 16.22-14.81l36.06-5.93a20.26 20.26 0 0 1 22.79 14.84l4.41 17.41c4.74-.33 9.52-.51 14.37-.51s9.63.18 14.37.51l4.41-17.41a20.25 20.25 0 0 1 22.79-14.84l36.06 5.93A20 20 0 0 1 221.85 57l29.53 116.38a19.93 19.93 0 0 1-11.31 23.18M227.28 176L199.23 65.46l-30.07-4.94l-2.84 11.17c2.9.58 5.78 1.2 8.61 1.92a12 12 0 1 1-5.86 23.27A168.4 168.4 0 0 0 128 92a168.4 168.4 0 0 0-41.07 4.88a12 12 0 0 1-5.86-23.27c2.83-.72 5.71-1.34 8.61-1.92l-2.83-11.17l-30.08 4.94L28.72 176l60.22 26.7l5-13.57c-4.37-.76-8.67-1.65-12.88-2.71a12 12 0 0 1 5.86-23.28A168.4 168.4 0 0 0 128 168a168.4 168.4 0 0 0 41.07-4.88a12 12 0 0 1 5.86 23.28c-4.21 1.06-8.51 1.95-12.88 2.71l5 13.57Z\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":19},"\u003Cpath fill=\"currentColor\" d=\"M212.62 75.17A63.7 63.7 0 0 0 206.39 26A12 12 0 0 0 196 20a63.71 63.71 0 0 0-50 24h-20a63.71 63.71 0 0 0-50-24a12 12 0 0 0-10.39 6a63.7 63.7 0 0 0-6.23 49.17A61.5 61.5 0 0 0 52 104v8a60.1 60.1 0 0 0 45.76 58.28A43.66 43.66 0 0 0 92 192v4H76a20 20 0 0 1-20-20a44.05 44.05 0 0 0-44-44a12 12 0 0 0 0 24a20 20 0 0 1 20 20a44.05 44.05 0 0 0 44 44h16v12a12 12 0 0 0 24 0v-40a20 20 0 0 1 40 0v40a12 12 0 0 0 24 0v-40a43.66 43.66 0 0 0-5.76-21.72A60.1 60.1 0 0 0 220 112v-8a61.5 61.5 0 0 0-7.38-28.83M196 112a36 36 0 0 1-36 36h-48a36 36 0 0 1-36-36v-8a37.87 37.87 0 0 1 6.13-20.12a11.65 11.65 0 0 0 1.58-11.49a39.9 39.9 0 0 1-.4-27.72a39.87 39.87 0 0 1 26.41 17.8a12 12 0 0 0 10.1 5.53h32.35a12 12 0 0 0 10.11-5.53a39.84 39.84 0 0 1 26.41-17.8a39.9 39.9 0 0 1-.4 27.72a12 12 0 0 0 1.61 11.53A37.85 37.85 0 0 1 196 104Z\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":21},"\u003Cpath fill=\"currentColor\" d=\"M167.31 160.69a16 16 0 1 1-22.62 0a16 16 0 0 1 22.62 0m-86.62-8a16 16 0 1 0 22.62 0a16 16 0 0 0-22.62 0m14.62-33.38a16 16 0 1 0-22.62 0a16 16 0 0 0 22.62 0m48-6.62a16 16 0 1 0 0 22.62a16 16 0 0 0 0-22.62M236 128A108 108 0 1 1 128 20a12 12 0 0 1 12 12a36 36 0 0 0 36 36a12 12 0 0 1 12 12a36 36 0 0 0 36 36a12 12 0 0 1 12 12m-24.67 10.65A60.17 60.17 0 0 1 165 91a60.17 60.17 0 0 1-47.66-46.32a84 84 0 1 0 94 94Z\"\u002F>",{"id":23,"title":24,"body":25,"category":3243,"date":3244,"description":3245,"extension":3246,"meta":3247,"navigation":3248,"path":3249,"readingTime":3250,"seo":3251,"seoDescription":3252,"seoTitle":3253,"stem":3254,"__hash__":3255},"blog_en\u002Fblog\u002Fscreen-mirror.md","Screen Mirror: Low-Latency Casting Architecture",{"type":26,"value":27,"toc":3184},"minimark",[28,33,110,113,122,125,136,141,144,170,173,177,184,315,338,341,345,348,430,433,482,486,492,615,640,644,658,662,672,737,741,779,798,801,808,812,818,917,932,938,942,1001,1004,1007,1030,1209,1212,1239,1243,1249,1273,1305,1320,1323,1327,1333,1339,1364,1368,1483,1489,1502,1505,1509,1524,1527,1531,1545,1664,1668,1674,1680,1717,1722,1749,1753,1768,1827,1831,1842,1877,1880,1884,1907,1913,1918,1941,1952,1956,1966,1972,1991,1997,2000,2004,2024,2028,2047,2115,2121,2125,2131,2209,2240,2244,2251,2327,2330,2336,2342,2346,2369,2498,2517,2521,2567,2683,2722,2725,2729,2768,2772,2791,2794,2798,2841,2845,2852,2871,2948,2952,2955,2959,2964,2967,3160,3164,3180],[29,30,32],"h2",{"id":31},"table-of-contents","Table of Contents",[34,35,36,44,50,56,62,68,74,80,86,92,98,104],"ul",{},[37,38,39],"li",{},[40,41,43],"a",{"href":42},"#high-level-architecture","High-Level Architecture",[37,45,46],{},[40,47,49],{"href":48},"#video-encoding-pipeline-android","Video Encoding Pipeline (Android)",[37,51,52],{},[40,53,55],{"href":54},"#videopacket-protocol-design","VideoPacket Protocol Design",[37,57,58],{},[40,59,61],{"href":60},"#video-decoding-pipeline-web","Video Decoding Pipeline (Web)",[37,63,64],{},[40,65,67],{"href":66},"#webgl2-rendering","WebGL2 Rendering",[37,69,70],{},[40,71,73],{"href":72},"#loss-detection--error-recovery","Loss Detection & Error Recovery",[37,75,76],{},[40,77,79],{"href":78},"#orientation-change-handling","Orientation Change Handling",[37,81,82],{},[40,83,85],{"href":84},"#system-mediaprojection-lifecycle","System MediaProjection Lifecycle",[37,87,88],{},[40,89,91],{"href":90},"#remote-control-touch-injection","Remote Control: Touch Injection",[37,93,94],{},[40,95,97],{"href":96},"#audio-pipeline","Audio Pipeline",[37,99,100],{},[40,101,103],{"href":102},"#performance-optimizations","Performance Optimizations",[37,105,106],{},[40,107,109],{"href":108},"#design-patterns-recap","Design Patterns Recap",[29,111,43],{"id":112},"high-level-architecture",[114,115,116,117,121],"p",{},"PlainApp screen mirror is an ",[118,119,120],"strong",{},"end-to-end low-latency casting system",": the Android device captures screen content, hardware-encodes it to H.264 video and Opus audio, and pushes it over WebSocket using a custom binary protocol to the web client; the web client decodes via the WebCodecs API and renders directly to a Canvas via WebGL2, with zero CPU copies throughout. A transparent touch overlay closes the loop, turning pointer input back into gestures on the phone.",[114,123,124],{},"No WebRTC, no RTMP, no intermediate server. The entire pipeline is:",[126,127,132],"pre",{"className":128,"code":130,"language":131},[129],"language-text","Android VirtualDisplay → MediaCodec H.264 Encoder → WebSocket →\nWebCodecs VideoDecoder → WebGL2 Texture → Canvas\n","text",[133,134,130],"code",{"__ignoreMap":135},"",[137,138,140],"h3",{"id":139},"why-not-webrtc","Why not WebRTC?",[114,142,143],{},"WebRTC is designed for real-time communication. Its ICE\u002FSTUN\u002FTURN negotiation, congestion control, and jitter buffering are overkill for LAN screen casting. PlainApp's use case is:",[34,145,146,152,158,164],{},[37,147,148,151],{},[118,149,150],{},"Same LAN",", latency \u003C 5ms, no NAT traversal needed",[37,153,154,157],{},[118,155,156],{},"Pursuing extreme low latency",", no jitter buffering",[37,159,160,163],{},[118,161,162],{},"High quality",", bitrate can be high (8Mbps)",[37,165,166,169],{},[118,167,168],{},"Screen control"," (touch injection), where WebRTC's DataChannel adds unnecessary complexity",[114,171,172],{},"A custom binary protocol over WebSocket is lighter and more controllable for the LAN scenario.",[137,174,176],{"id":175},"component-map","Component map",[114,178,179],{},[180,181],"img",{"alt":182,"src":183},"Diagram 1","\u002Fblog\u002Fscreen-mirror\u002Fdiagram-01.svg",[185,186,187,203],"table",{},[188,189,190],"thead",{},[191,192,193,197,200],"tr",{},[194,195,196],"th",{},"Layer",[194,198,199],{},"Android",[194,201,202],{},"Web",[204,205,206,224,237,249,260,272,288,301],"tbody",{},[191,207,208,212,221],{},[209,210,211],"td",{},"Screen capture",[209,213,214,217,218],{},[133,215,216],{},"MediaProjection"," + ",[133,219,220],{},"VirtualDisplay",[209,222,223],{},"—",[191,225,226,229,235],{},[209,227,228],{},"Video encoding",[209,230,231,234],{},[133,232,233],{},"MediaCodec"," H.264 hardware encoder",[209,236,223],{},[191,238,239,242,247],{},[209,240,241],{},"Audio encoding",[209,243,244,246],{},[133,245,233],{}," Opus hardware encoder",[209,248,223],{},[191,250,251,254,257],{},[209,252,253],{},"Transport",[209,255,256],{},"WebSocket binary events",[209,258,259],{},"WebSocket receiver",[191,261,262,265,267],{},[209,263,264],{},"Video decoding",[209,266,223],{},[209,268,269],{},[133,270,271],{},"WebCodecs VideoDecoder",[191,273,274,277,279],{},[209,275,276],{},"Audio decoding",[209,278,223],{},[209,280,281,284,285],{},[133,282,283],{},"WebCodecs AudioDecoder"," → ",[133,286,287],{},"\u003Caudio>",[191,289,290,293,295],{},[209,291,292],{},"Rendering",[209,294,223],{},[209,296,297,300],{},[133,298,299],{},"WebGL2"," texture direct-render",[191,302,303,306,312],{},[209,304,305],{},"Control",[209,307,308,311],{},[133,309,310],{},"AccessibilityService"," gesture injection",[209,313,314],{},"Touch overlay → GraphQL mutation",[114,316,317,318,321,322,325,326,329,330,333,334,337],{},"Video and audio always flow ",[118,319,320],{},"device → browser"," over the same WebSocket connection; control flows the opposite direction over ",[118,323,324],{},"GraphQL"," (",[133,327,328],{},"sendScreenMirrorControl","), which also doubles as the side-channel for the codec config (",[133,331,332],{},"screenMirrorVideoCodec"," query) and keyframe requests (",[133,335,336],{},"requestScreenMirrorKeyFrame"," mutation).",[29,339,49],{"id":340},"video-encoding-pipeline-android",[137,342,344],{"id":343},"encoding-parameter-tuning","Encoding Parameter Tuning",[114,346,347],{},"Encoding parameters were tuned specifically for low-latency LAN screen casting:",[185,349,350,363],{},[188,351,352],{},[191,353,354,357,360],{},[194,355,356],{},"Parameter",[194,358,359],{},"Value",[194,361,362],{},"Notes",[204,364,365,378,391,404,417],{},[191,366,367,372,375],{},[209,368,369],{},[133,370,371],{},"KEY_FRAME_RATE",[209,373,374],{},"60",[209,376,377],{},"60fps for smoothness",[191,379,380,385,388],{},[209,381,382],{},[133,383,384],{},"KEY_I_FRAME_INTERVAL",[209,386,387],{},"10",[209,389,390],{},"IDR interval 10s, reduces keyframe overhead",[191,392,393,398,401],{},[209,394,395],{},[133,396,397],{},"KEY_BIT_RATE_MODE",[209,399,400],{},"VBR (implicit, no explicit mode set)",[209,402,403],{},"Variable bitrate, scene-adaptive",[191,405,406,411,414],{},[209,407,408],{},[133,409,410],{},"KEY_PRIORITY",[209,412,413],{},"0",[209,415,416],{},"Realtime priority",[191,418,419,424,427],{},[209,420,421],{},[133,422,423],{},"KEY_LATENCY",[209,425,426],{},"1",[209,428,429],{},"Low-latency mode",[114,431,432],{},"Bitrate is tiered by quality mode — higher bitrates (e.g. 24 Mbps) were tested and caused encoder\u002Fdecoder frame drops and increased end-to-end latency without visible quality gain for screen content:",[185,434,435,448],{},[188,436,437],{},[191,438,439,442,445],{},[194,440,441],{},"Mode",[194,443,444],{},"Bitrate",[194,446,447],{},"Capture resolution",[204,449,450,461,471],{},[191,451,452,455,458],{},[209,453,454],{},"HD",[209,456,457],{},"8 Mbps",[209,459,460],{},"1080p short side",[191,462,463,466,469],{},[209,464,465],{},"Smooth",[209,467,468],{},"4 Mbps",[209,470,460],{},[191,472,473,476,479],{},[209,474,475],{},"Low",[209,477,478],{},"2 Mbps",[209,480,481],{},"720p short side",[137,483,485],{"id":484},"encoder-low-latency-configuration","Encoder Low-Latency Configuration",[114,487,488,491],{},[133,489,490],{},"MediaCodecVideoEncoder"," configures the encoder once at creation time:",[126,493,497],{"className":494,"code":495,"language":496,"meta":135,"style":135},"language-kotlin shiki shiki-themes github-light github-dark","MediaFormat.createVideoFormat(MIME, width, height).apply {\n    setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface)\n    setInteger(MediaFormat.KEY_BIT_RATE, bitrateBps)\n    setInteger(MediaFormat.KEY_FRAME_RATE, frameRate)          \u002F\u002F 60\n    setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, iFrameIntervalSec) \u002F\u002F 10\n    setLong(MediaFormat.KEY_REPEAT_PREVIOUS_FRAME_AFTER, 100_000L)\n    setInteger(MediaFormat.KEY_COLOR_RANGE, MediaFormat.COLOR_RANGE_LIMITED)\n    setInteger(MediaFormat.KEY_PRIORITY, 0)\n    setInteger(MediaFormat.KEY_LATENCY, 1)\n}\n","kotlin",[133,498,499,521,530,538,550,561,577,585,597,609],{"__ignoreMap":135},[500,501,504,508,512,515,518],"span",{"class":502,"line":503},"line",1,[500,505,507],{"class":506},"sVt8B","MediaFormat.",[500,509,511],{"class":510},"sScJk","createVideoFormat",[500,513,514],{"class":506},"(MIME, width, height).",[500,516,517],{"class":510},"apply",[500,519,520],{"class":506}," {\n",[500,522,524,527],{"class":502,"line":523},2,[500,525,526],{"class":510},"    setInteger",[500,528,529],{"class":506},"(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface)\n",[500,531,533,535],{"class":502,"line":532},3,[500,534,526],{"class":510},[500,536,537],{"class":506},"(MediaFormat.KEY_BIT_RATE, bitrateBps)\n",[500,539,541,543,546],{"class":502,"line":540},4,[500,542,526],{"class":510},[500,544,545],{"class":506},"(MediaFormat.KEY_FRAME_RATE, frameRate)          ",[500,547,549],{"class":548},"sJ8bj","\u002F\u002F 60\n",[500,551,553,555,558],{"class":502,"line":552},5,[500,554,526],{"class":510},[500,556,557],{"class":506},"(MediaFormat.KEY_I_FRAME_INTERVAL, iFrameIntervalSec) ",[500,559,560],{"class":548},"\u002F\u002F 10\n",[500,562,564,567,570,574],{"class":502,"line":563},6,[500,565,566],{"class":510},"    setLong",[500,568,569],{"class":506},"(MediaFormat.KEY_REPEAT_PREVIOUS_FRAME_AFTER, ",[500,571,573],{"class":572},"sj4cs","100_000L",[500,575,576],{"class":506},")\n",[500,578,580,582],{"class":502,"line":579},7,[500,581,526],{"class":510},[500,583,584],{"class":506},"(MediaFormat.KEY_COLOR_RANGE, MediaFormat.COLOR_RANGE_LIMITED)\n",[500,586,588,590,593,595],{"class":502,"line":587},8,[500,589,526],{"class":510},[500,591,592],{"class":506},"(MediaFormat.KEY_PRIORITY, ",[500,594,413],{"class":572},[500,596,576],{"class":506},[500,598,600,602,605,607],{"class":502,"line":599},9,[500,601,526],{"class":510},[500,603,604],{"class":506},"(MediaFormat.KEY_LATENCY, ",[500,606,426],{"class":572},[500,608,576],{"class":506},[500,610,612],{"class":502,"line":611},10,[500,613,614],{"class":506},"}\n",[114,616,617,620,621,624,625,628,629,632,633,635,636,639],{},[133,618,619],{},"KEY_PRIORITY=0"," and ",[133,622,623],{},"KEY_LATENCY=1"," are the keys to low latency — they tell the encoder to prioritize real-time encoding over compression ratio. The input is a ",[133,626,627],{},"Surface"," created by ",[133,630,631],{},"MediaCodec.createInputSurface()"," and fed directly to ",[133,634,220],{}," — no ",[133,637,638],{},"SurfaceTexture"," readback, no I420 conversion, no CPU touches the pixels.",[137,641,643],{"id":642},"capture-resolution","Capture Resolution",[114,645,646,649,650,653,654,657],{},[133,647,648],{},"ScreenMirrorCaptureSize.compute()"," derives the actual capture size from the physical screen size, the quality mode's short-side target (720\u002F1080), and the encoder's reported ",[133,651,652],{},"maxWidth\u002FmaxHeight"," and width\u002Fheight alignment (queried once via ",[133,655,656],{},"MediaCodecVideoEncoder.queryEncoderCaps()","), so the encoder never receives dimensions it can't accept.",[137,659,661],{"id":660},"keyframe-requests","Keyframe Requests",[114,663,664,665,667,668,671],{},"The web client can request an IDR frame via the GraphQL ",[133,666,336],{}," mutation to recover from packet loss. Android responds via ",[133,669,670],{},"MediaCodec.PARAMETER_KEY_REQUEST_SYNC_FRAME",":",[126,673,675],{"className":494,"code":674,"language":496,"meta":135,"style":135},"fun requestKeyFrame() {\n    val b = Bundle().apply { putInt(MediaCodec.PARAMETER_KEY_REQUEST_SYNC_FRAME, 1) }\n    codec?.setParameters(b)\n}\n",[133,676,677,689,722,733],{"__ignoreMap":135},[500,678,679,683,686],{"class":502,"line":503},[500,680,682],{"class":681},"szBVR","fun",[500,684,685],{"class":510}," requestKeyFrame",[500,687,688],{"class":506},"() {\n",[500,690,691,694,697,700,703,706,708,711,714,717,719],{"class":502,"line":523},[500,692,693],{"class":681},"    val",[500,695,696],{"class":506}," b ",[500,698,699],{"class":681},"=",[500,701,702],{"class":510}," Bundle",[500,704,705],{"class":506},"().",[500,707,517],{"class":510},[500,709,710],{"class":506}," { ",[500,712,713],{"class":510},"putInt",[500,715,716],{"class":506},"(MediaCodec.PARAMETER_KEY_REQUEST_SYNC_FRAME, ",[500,718,426],{"class":572},[500,720,721],{"class":506},") }\n",[500,723,724,727,730],{"class":502,"line":532},[500,725,726],{"class":506},"    codec?.",[500,728,729],{"class":510},"setParameters",[500,731,732],{"class":506},"(b)\n",[500,734,735],{"class":502,"line":540},[500,736,614],{"class":506},[137,738,740],{"id":739},"spspps-and-keyframe-broadcast","SPS\u002FPPS and Keyframe Broadcast",[114,742,743,744,747,748,751,752,755,756,759,760,763,764,767,768,770,771,774,775,778],{},"After the encoder starts, ",[133,745,746],{},"INFO_OUTPUT_FORMAT_CHANGED"," delivers ",[133,749,750],{},"csd-0","\u002F",[133,753,754],{},"csd-1"," (SPS\u002FPPS), which ",[133,757,758],{},"ScreenMirrorPipeline"," joins into a single Annex-B config blob and caches (",[133,761,762],{},"cachedConfig","). The first IDR that follows is cached too (",[133,765,766],{},"cachedKeyFrame",") so a freshly-connected web client can pull both via the ",[133,769,332],{}," GraphQL query without waiting for the next keyframe interval. When the config just changed (orientation or quality switch), Android does ",[118,772,773],{},"not"," send the new IDR as a normal video packet — it bundles SPS\u002FPPS + IDR into one ",[133,776,777],{},"screen_mirror_video_codec"," WebSocket event, so the web client completes decoder reconfiguration and first-frame decoding in one shot instead of racing a stale decoder against a new bitstream.",[114,780,781,782,620,785,788,789,793,794,797],{},"Some OEM encoders (Qualcomm\u002FXiaomi) bundle SPS+PPS+IDR into a single output buffer carrying both ",[133,783,784],{},"BUFFER_FLAG_CODEC_CONFIG",[133,786,787],{},"BUFFER_FLAG_SYNC_FRAME",". The drain loop only skips buffers that are ",[790,791,792],"em",{},"pure"," config (",[133,795,796],{},"isConfig && !isKey",") — skipping a config-flagged buffer that also carries the sync frame would silently drop the IDR and leave the decoder with only P-frames, producing mosaic output.",[29,799,55],{"id":800},"videopacket-protocol-design",[114,802,803,804,807],{},"Both video and audio frames are wrapped in the unified ",[133,805,806],{},"VideoPacket"," binary protocol for WebSocket transport.",[137,809,811],{"id":810},"protocol-format","Protocol Format",[126,813,816],{"className":814,"code":815,"language":131},[129],"+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n| MAGIC  | FLAGS  |           FRAME_ID (4 bytes, big-endian)            |              TIMESTAMP (8 bytes, BE)              |  DATA  |\n| 0x56   |        |   byte2   |   byte3   |   byte4   |   byte5   |  byte6  |  byte7  | ...  |  byte13 |        payload...        |\n+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+--------+\n \\- 1B -\u002F \\- 1B -\u002F \\-------------------- 4B ----------------------\u002F \\----------------------- 8B ------------------------\u002F \\- var -\u002F\n",[133,817,815],{"__ignoreMap":135},[185,819,820,833],{},[188,821,822],{},[191,823,824,827,830],{},[194,825,826],{},"Field",[194,828,829],{},"Size",[194,831,832],{},"Description",[204,834,835,855,878,891,904],{},[191,836,837,842,845],{},[209,838,839],{},[133,840,841],{},"MAGIC",[209,843,844],{},"1 byte",[209,846,847,848,325,851,854],{},"Fixed ",[133,849,850],{},"0x56",[133,852,853],{},"'V'","), for validation",[191,856,857,862,864],{},[209,858,859],{},[133,860,861],{},"FLAGS",[209,863,844],{},[209,865,866,869,870,873,874,877],{},[133,867,868],{},"0x01","=keyframe, ",[133,871,872],{},"0x02","=config, ",[133,875,876],{},"0x04","=audio",[191,879,880,885,888],{},[209,881,882],{},[133,883,884],{},"FRAME_ID",[209,886,887],{},"4 bytes",[209,889,890],{},"Monotonically increasing frame number, uint32 big-endian",[191,892,893,898,901],{},[209,894,895],{},[133,896,897],{},"TIMESTAMP",[209,899,900],{},"8 bytes",[209,902,903],{},"Encoder PTS in microseconds, big-endian",[191,905,906,911,914],{},[209,907,908],{},[133,909,910],{},"DATA",[209,912,913],{},"variable",[209,915,916],{},"H.264 NAL unit or Opus data",[114,918,919,920,923,924,927,928,931],{},"Both Android's ",[133,921,922],{},"VideoPacket.encode()"," (in ",[133,925,926],{},"commonMain",", so its wire format is covered by JVM unit tests without any Android dependency) and the web's ",[133,929,930],{},"parseVideoPacket()"," implement this format independently — there's no shared serialization library, just a spec both sides honor.",[114,933,934],{},[180,935],{"alt":936,"src":937},"Diagram 2","\u002Fblog\u002Fscreen-mirror\u002Fdiagram-02.svg",[137,939,941],{"id":940},"design-notes","Design Notes",[34,943,944,964,979,987],{},[37,945,946,951,952,955,956,959,960,963],{},[118,947,948,950],{},[133,949,884],{}," unsigned parsing",": ",[133,953,954],{},"((buf[2] \u003C\u003C 24) | (buf[3] \u003C\u003C 16) | (buf[4] \u003C\u003C 8) | buf[5]) >>> 0"," — must use ",[133,957,958],{},">>> 0"," to ensure unsigned, otherwise ",[133,961,962],{},"frameId > 2^31"," is parsed as negative, causing false loss detection.",[37,965,966,971,972,975,976,978],{},[118,967,968,970],{},[133,969,884],{}," never resets",": When the encoder is rebuilt for orientation change, ",[133,973,974],{},"frameId"," continues incrementing (it lives in ",[133,977,758],{},", not in the encoder). This lets the web side detect frame loss during rotation via frameId gaps.",[37,980,981,986],{},[118,982,983,985],{},[133,984,897],{}," uses encoder PTS",": No dependence on the web client's clock, avoiding clock drift causing A\u002FV desync.",[37,988,989,992,993,996,997,1000],{},[118,990,991],{},"Zero-copy parsing",": the web parser slices the payload with ",[133,994,995],{},"Uint8Array.subarray()"," — a view into the original WebSocket ",[133,998,999],{},"ArrayBuffer",", not a copy.",[29,1002,61],{"id":1003},"video-decoding-pipeline-web",[137,1005,271],{"id":1006},"webcodecs-videodecoder",[114,1008,1009,1010,1013,1014,1017,1018,1021,1022,1025,1026,1029],{},"The web client uses ",[133,1011,1012],{},"WebCodecs API","'s ",[133,1015,1016],{},"VideoDecoder"," for hardware decoding. Compared to ",[133,1019,1020],{},"MediaSource Extensions"," or ",[133,1023,1024],{},"WebRTC",", WebCodecs provides fine-grained control over the decoding process — no jitter buffer, no container layer, and decoded ",[133,1027,1028],{},"VideoFrame"," objects can be directly uploaded as WebGL textures.",[126,1031,1035],{"className":1032,"code":1033,"language":1034,"meta":135,"style":135},"language-typescript shiki shiki-themes github-light github-dark","const decoder = new VideoDecoder({\n    output: (frame) => this.renderFrame(frame),\n    error: (e) => {\n        this.waitingForIdr = true\n        this.onRequestKeyFrame?.()\n        this.onError?.(e)\n    },\n})\ndecoder.configure({\n    codec,                              \u002F\u002F e.g. 'avc1.42c01e', read from the SPS NAL\n    avc: { format: 'annexb' },\n    optimizeForLatency: true,\n    hardwareAcceleration: 'prefer-hardware',\n})\n","typescript",[133,1036,1037,1057,1087,1103,1116,1128,1140,1145,1150,1160,1168,1181,1193,1204],{"__ignoreMap":135},[500,1038,1039,1042,1045,1048,1051,1054],{"class":502,"line":503},[500,1040,1041],{"class":681},"const",[500,1043,1044],{"class":572}," decoder",[500,1046,1047],{"class":681}," =",[500,1049,1050],{"class":681}," new",[500,1052,1053],{"class":510}," VideoDecoder",[500,1055,1056],{"class":506},"({\n",[500,1058,1059,1062,1065,1069,1072,1075,1078,1081,1084],{"class":502,"line":523},[500,1060,1061],{"class":510},"    output",[500,1063,1064],{"class":506},": (",[500,1066,1068],{"class":1067},"s4XuR","frame",[500,1070,1071],{"class":506},") ",[500,1073,1074],{"class":681},"=>",[500,1076,1077],{"class":572}," this",[500,1079,1080],{"class":506},".",[500,1082,1083],{"class":510},"renderFrame",[500,1085,1086],{"class":506},"(frame),\n",[500,1088,1089,1092,1094,1097,1099,1101],{"class":502,"line":532},[500,1090,1091],{"class":510},"    error",[500,1093,1064],{"class":506},[500,1095,1096],{"class":1067},"e",[500,1098,1071],{"class":506},[500,1100,1074],{"class":681},[500,1102,520],{"class":506},[500,1104,1105,1108,1111,1113],{"class":502,"line":540},[500,1106,1107],{"class":572},"        this",[500,1109,1110],{"class":506},".waitingForIdr ",[500,1112,699],{"class":681},[500,1114,1115],{"class":572}," true\n",[500,1117,1118,1120,1122,1125],{"class":502,"line":552},[500,1119,1107],{"class":572},[500,1121,1080],{"class":506},[500,1123,1124],{"class":510},"onRequestKeyFrame",[500,1126,1127],{"class":506},"?.()\n",[500,1129,1130,1132,1134,1137],{"class":502,"line":563},[500,1131,1107],{"class":572},[500,1133,1080],{"class":506},[500,1135,1136],{"class":510},"onError",[500,1138,1139],{"class":506},"?.(e)\n",[500,1141,1142],{"class":502,"line":579},[500,1143,1144],{"class":506},"    },\n",[500,1146,1147],{"class":502,"line":587},[500,1148,1149],{"class":506},"})\n",[500,1151,1152,1155,1158],{"class":502,"line":599},[500,1153,1154],{"class":506},"decoder.",[500,1156,1157],{"class":510},"configure",[500,1159,1056],{"class":506},[500,1161,1162,1165],{"class":502,"line":611},[500,1163,1164],{"class":506},"    codec,                              ",[500,1166,1167],{"class":548},"\u002F\u002F e.g. 'avc1.42c01e', read from the SPS NAL\n",[500,1169,1171,1174,1178],{"class":502,"line":1170},11,[500,1172,1173],{"class":506},"    avc: { format: ",[500,1175,1177],{"class":1176},"sZZnC","'annexb'",[500,1179,1180],{"class":506}," },\n",[500,1182,1184,1187,1190],{"class":502,"line":1183},12,[500,1185,1186],{"class":506},"    optimizeForLatency: ",[500,1188,1189],{"class":572},"true",[500,1191,1192],{"class":506},",\n",[500,1194,1196,1199,1202],{"class":502,"line":1195},13,[500,1197,1198],{"class":506},"    hardwareAcceleration: ",[500,1200,1201],{"class":1176},"'prefer-hardware'",[500,1203,1192],{"class":506},[500,1205,1207],{"class":502,"line":1206},14,[500,1208,1149],{"class":506},[114,1210,1211],{},"Key configurations:",[34,1213,1214,1220,1226,1232],{},[37,1215,1216,1219],{},[133,1217,1218],{},"optimizeForLatency: true"," — tells the decoder to prioritize low latency, no frame buffering",[37,1221,1222,1225],{},[133,1223,1224],{},"hardwareAcceleration: 'prefer-hardware'"," — prefer GPU decoding",[37,1227,1228,1231],{},[133,1229,1230],{},"avc: { format: 'annexb' }"," — use Annex-B format with inline SPS\u002FPPS before each IDR",[37,1233,1234,1235,1238],{},"the codec string itself isn't hardcoded — ",[133,1236,1237],{},"extractAvc1CodecString()"," reads profile\u002Fcompat\u002Flevel bytes directly out of the first SPS NAL in the config blob",[137,1240,1242],{"id":1241},"green-screen-problem-and-startup-sequence","Green Screen Problem and Startup Sequence",[114,1244,1245,1246,1248],{},"The encoder produces its first IDR frame before ",[133,1247,220],{}," has rendered real screen content — it's a blank (green) frame. If the web side decodes this frame, the user sees a green flash until screen content changes and triggers a new frame.",[114,1250,1251,1254,1255,1257,1258,1261,1262,1265,1266,1269,1270,1272],{},[118,1252,1253],{},"Solution",": On startup, the web side pulls the cached config via the ",[133,1256,332],{}," GraphQL query but does not decode the bundled keyframe. Instead, it calls ",[133,1259,1260],{},"video.requestIdr()"," to set ",[133,1263,1264],{},"waitingForIdr = true"," (dropping all P-frames until an IDR arrives), then calls ",[133,1267,1268],{},"requestKeyFrame()"," to request a fresh IDR over the same mutation used for loss recovery. By the time the new IDR arrives, ",[133,1271,220],{}," has real screen content.",[126,1274,1276],{"className":1032,"code":1275,"language":1034,"meta":135,"style":135},"video.requestIdr()      \u002F\u002F drop P-frames, wait for IDR\nawait requestKeyFrame() \u002F\u002F request fresh IDR via GraphQL mutation\n",[133,1277,1278,1292],{"__ignoreMap":135},[500,1279,1280,1283,1286,1289],{"class":502,"line":503},[500,1281,1282],{"class":506},"video.",[500,1284,1285],{"class":510},"requestIdr",[500,1287,1288],{"class":506},"()      ",[500,1290,1291],{"class":548},"\u002F\u002F drop P-frames, wait for IDR\n",[500,1293,1294,1297,1299,1302],{"class":502,"line":523},[500,1295,1296],{"class":681},"await",[500,1298,685],{"class":510},[500,1300,1301],{"class":506},"() ",[500,1303,1304],{"class":548},"\u002F\u002F request fresh IDR via GraphQL mutation\n",[114,1306,1307,1308,1311,1312,1315,1316,1319],{},"The ",[133,1309,1310],{},"onFirstFrameRendered"," callback is bound to ",[133,1313,1314],{},"renderFrame()"," rather than ",[133,1317,1318],{},"handleVideo()",", ensuring the UI only updates after a real frame is rendered — not merely received.",[29,1321,67],{"id":1322},"webgl2-rendering",[137,1324,1326],{"id":1325},"zero-copy-gpu-direct-render","Zero-Copy GPU Direct Render",[114,1328,1329,1330,1332],{},"Decoded ",[133,1331,1028],{}," objects are directly uploaded as WebGL2 textures, never passing through the CPU:",[126,1334,1337],{"className":1335,"code":1336,"language":131},[129],"VideoDecoder → VideoFrame → gl.texImage2D(VideoFrame) → Canvas\n",[133,1338,1336],{"__ignoreMap":135},[114,1340,1341,1344,1345,1347,1348,1351,1352,1355,1356,1359,1360,1363],{},[133,1342,1343],{},"gl.texImage2D"," accepts ",[133,1346,1028],{}," as a pixel source. The browser handles YUV→RGB conversion and GPU upload internally — no ",[133,1349,1350],{},"ImageData"," CPU copy. ",[133,1353,1354],{},"MirrorGLRenderer"," falls back to Canvas 2D ",[133,1357,1358],{},"drawImage()"," if ",[133,1361,1362],{},"getContext('webgl2', ...)"," fails, so older browsers still get a (slightly higher-latency) picture.",[137,1365,1367],{"id":1366},"desynchronized-context","desynchronized Context",[126,1369,1371],{"className":1032,"code":1370,"language":1034,"meta":135,"style":135},"const gl = canvas.getContext('webgl2', {\n    alpha: false,\n    desynchronized: true,        \u002F\u002F bypass compositor, write directly to screen\n    preserveDrawingBuffer: true, \u002F\u002F preserve buffer for screenshots\n    powerPreference: 'high-performance',\n    antialias: false,\n    depth: false,\n    stencil: false,\n    premultipliedAlpha: false,\n})\n",[133,1372,1373,1397,1407,1420,1433,1443,1452,1461,1470,1479],{"__ignoreMap":135},[500,1374,1375,1377,1380,1382,1385,1388,1391,1394],{"class":502,"line":503},[500,1376,1041],{"class":681},[500,1378,1379],{"class":572}," gl",[500,1381,1047],{"class":681},[500,1383,1384],{"class":506}," canvas.",[500,1386,1387],{"class":510},"getContext",[500,1389,1390],{"class":506},"(",[500,1392,1393],{"class":1176},"'webgl2'",[500,1395,1396],{"class":506},", {\n",[500,1398,1399,1402,1405],{"class":502,"line":523},[500,1400,1401],{"class":506},"    alpha: ",[500,1403,1404],{"class":572},"false",[500,1406,1192],{"class":506},[500,1408,1409,1412,1414,1417],{"class":502,"line":532},[500,1410,1411],{"class":506},"    desynchronized: ",[500,1413,1189],{"class":572},[500,1415,1416],{"class":506},",        ",[500,1418,1419],{"class":548},"\u002F\u002F bypass compositor, write directly to screen\n",[500,1421,1422,1425,1427,1430],{"class":502,"line":540},[500,1423,1424],{"class":506},"    preserveDrawingBuffer: ",[500,1426,1189],{"class":572},[500,1428,1429],{"class":506},", ",[500,1431,1432],{"class":548},"\u002F\u002F preserve buffer for screenshots\n",[500,1434,1435,1438,1441],{"class":502,"line":552},[500,1436,1437],{"class":506},"    powerPreference: ",[500,1439,1440],{"class":1176},"'high-performance'",[500,1442,1192],{"class":506},[500,1444,1445,1448,1450],{"class":502,"line":563},[500,1446,1447],{"class":506},"    antialias: ",[500,1449,1404],{"class":572},[500,1451,1192],{"class":506},[500,1453,1454,1457,1459],{"class":502,"line":579},[500,1455,1456],{"class":506},"    depth: ",[500,1458,1404],{"class":572},[500,1460,1192],{"class":506},[500,1462,1463,1466,1468],{"class":502,"line":587},[500,1464,1465],{"class":506},"    stencil: ",[500,1467,1404],{"class":572},[500,1469,1192],{"class":506},[500,1471,1472,1475,1477],{"class":502,"line":599},[500,1473,1474],{"class":506},"    premultipliedAlpha: ",[500,1476,1404],{"class":572},[500,1478,1192],{"class":506},[500,1480,1481],{"class":502,"line":611},[500,1482,1149],{"class":506},[114,1484,1485,1488],{},[133,1486,1487],{},"desynchronized: true"," bypasses the browser compositor, writing directly to the screen, saving ~1 frame of display latency (~16ms @ 60fps).",[114,1490,1491,1494,1495,1498,1499,1501],{},[133,1492,1493],{},"preserveDrawingBuffer: true"," preserves the drawing buffer so ",[133,1496,1497],{},"canvas.toDataURL()"," screenshots can read the content. With the default ",[133,1500,1404],{},", the buffer is cleared after compositing, producing black screenshots.",[114,1503,1504],{},"The shader itself is deliberately minimal — a fullscreen-triangle vertex shader and a one-line fragment shader that samples the texture — because the only work needed per frame is \"put this texture on the screen.\"",[137,1506,1508],{"id":1507},"canvas-auto-fit","Canvas Auto-Fit",[114,1510,1511,1512,1515,1516,1519,1520,1523],{},"The canvas backing store size is set from ",[133,1513,1514],{},"VideoFrame.displayWidth\u002FHeight"," whenever it changes. The CSS size is then fit to the wrapper container by ",[133,1517,1518],{},"fitCanvasToWrapper()"," while preserving aspect ratio (letterboxing or pillarboxing as needed). A ",[133,1521,1522],{},"ResizeObserver"," on the canvas's parent element re-runs this fit whenever the container resizes, so the video never stretches.",[29,1525,73],{"id":1526},"loss-detection-error-recovery",[137,1528,1530],{"id":1529},"frameid-gap-detection","FrameId Gap Detection",[114,1532,1533,1534,1536,1537,1540,1541,1544],{},"Each video frame carries a monotonically increasing ",[133,1535,974],{},". The decoder tracks ",[133,1538,1539],{},"lastFrameId","; if a new frame's ",[133,1542,1543],{},"frameId > lastFrameId + 1",", frames were lost:",[126,1546,1548],{"className":1032,"code":1547,"language":1034,"meta":135,"style":135},"if (!this.waitingForIdr && this.lastFrameId > 0\n    && packet.frameId > this.lastFrameId + 1) {\n    if (!packet.isKeyFrame) {\n        \u002F\u002F Loss: drop subsequent P-frames, request new IDR\n        this.waitingForIdr = true\n        this.onRequestKeyFrame?.()\n        this.lastFrameId = packet.frameId\n        return\n    }\n}\n",[133,1549,1550,1579,1602,1614,1619,1629,1639,1650,1655,1660],{"__ignoreMap":135},[500,1551,1552,1555,1557,1560,1563,1565,1568,1570,1573,1576],{"class":502,"line":503},[500,1553,1554],{"class":681},"if",[500,1556,325],{"class":506},[500,1558,1559],{"class":681},"!",[500,1561,1562],{"class":572},"this",[500,1564,1110],{"class":506},[500,1566,1567],{"class":681},"&&",[500,1569,1077],{"class":572},[500,1571,1572],{"class":506},".lastFrameId ",[500,1574,1575],{"class":681},">",[500,1577,1578],{"class":572}," 0\n",[500,1580,1581,1584,1587,1589,1591,1593,1596,1599],{"class":502,"line":523},[500,1582,1583],{"class":681},"    &&",[500,1585,1586],{"class":506}," packet.frameId ",[500,1588,1575],{"class":681},[500,1590,1077],{"class":572},[500,1592,1572],{"class":506},[500,1594,1595],{"class":681},"+",[500,1597,1598],{"class":572}," 1",[500,1600,1601],{"class":506},") {\n",[500,1603,1604,1607,1609,1611],{"class":502,"line":532},[500,1605,1606],{"class":681},"    if",[500,1608,325],{"class":506},[500,1610,1559],{"class":681},[500,1612,1613],{"class":506},"packet.isKeyFrame) {\n",[500,1615,1616],{"class":502,"line":540},[500,1617,1618],{"class":548},"        \u002F\u002F Loss: drop subsequent P-frames, request new IDR\n",[500,1620,1621,1623,1625,1627],{"class":502,"line":552},[500,1622,1107],{"class":572},[500,1624,1110],{"class":506},[500,1626,699],{"class":681},[500,1628,1115],{"class":572},[500,1630,1631,1633,1635,1637],{"class":502,"line":563},[500,1632,1107],{"class":572},[500,1634,1080],{"class":506},[500,1636,1124],{"class":510},[500,1638,1127],{"class":506},[500,1640,1641,1643,1645,1647],{"class":502,"line":579},[500,1642,1107],{"class":572},[500,1644,1572],{"class":506},[500,1646,699],{"class":681},[500,1648,1649],{"class":506}," packet.frameId\n",[500,1651,1652],{"class":502,"line":587},[500,1653,1654],{"class":681},"        return\n",[500,1656,1657],{"class":502,"line":599},[500,1658,1659],{"class":506},"    }\n",[500,1661,1662],{"class":502,"line":611},[500,1663,614],{"class":506},[137,1665,1667],{"id":1666},"waitingforidr-state-machine","waitingForIdr State Machine",[114,1669,1670,1673],{},[133,1671,1672],{},"waitingForIdr"," is a simple two-state machine:",[114,1675,1676],{},[180,1677],{"alt":1678,"src":1679},"Diagram 3","\u002Fblog\u002Fscreen-mirror\u002Fdiagram-03.svg",[185,1681,1682,1692],{},[188,1683,1684],{},[191,1685,1686,1689],{},[194,1687,1688],{},"State",[194,1690,1691],{},"Behavior",[204,1693,1694,1704],{},[191,1695,1696,1701],{},[209,1697,1698],{},[133,1699,1700],{},"NORMAL",[209,1702,1703],{},"Decode all frames normally",[191,1705,1706,1711],{},[209,1707,1708],{},[133,1709,1710],{},"WAITING_FOR_IDR",[209,1712,1713,1714,1716],{},"Drop all P-frames, only decode IDR frames; reset to ",[133,1715,1700],{}," when IDR arrives",[114,1718,1719,1720,671],{},"Scenarios that trigger the transition into ",[133,1721,1710],{},[1723,1724,1725,1731,1737,1743],"ol",{},[37,1726,1727,1730],{},[118,1728,1729],{},"At startup",": skip stale GraphQL keyframe, wait for real IDR",[37,1732,1733,1736],{},[118,1734,1735],{},"On packet loss",": drop undecodable P-frames, wait for IDR recovery",[37,1738,1739,1742],{},[118,1740,1741],{},"On decoder error",": reset decoder, wait for IDR",[37,1744,1745,1748],{},[118,1746,1747],{},"On config change",": drop residual P-frames after orientation\u002Fquality change",[137,1750,1752],{"id":1751},"decoder-error-recovery","Decoder Error Recovery",[114,1754,1755,1756,1759,1760,1763,1764,1767],{},"When ",[133,1757,1758],{},"VideoDecoder.onerror"," fires, ",[133,1761,1762],{},"decoderNeedsReset = true"," is set in the pipeline layer (",[133,1765,1766],{},"screen-mirror-pipeline.ts","). On the next IDR frame, the decoder is reconfigured with the cached SPS\u002FPPS instead of round-tripping to GraphQL again:",[126,1769,1771],{"className":1032,"code":1770,"language":1034,"meta":135,"style":135},"if (decoderNeedsReset) {\n    if (!packet.isKeyFrame || !cachedConfig) return\n    video.configure(cachedConfig)\n    decoderNeedsReset = false\n}\n",[133,1772,1773,1780,1803,1813,1823],{"__ignoreMap":135},[500,1774,1775,1777],{"class":502,"line":503},[500,1776,1554],{"class":681},[500,1778,1779],{"class":506}," (decoderNeedsReset) {\n",[500,1781,1782,1784,1786,1788,1791,1794,1797,1800],{"class":502,"line":523},[500,1783,1606],{"class":681},[500,1785,325],{"class":506},[500,1787,1559],{"class":681},[500,1789,1790],{"class":506},"packet.isKeyFrame ",[500,1792,1793],{"class":681},"||",[500,1795,1796],{"class":681}," !",[500,1798,1799],{"class":506},"cachedConfig) ",[500,1801,1802],{"class":681},"return\n",[500,1804,1805,1808,1810],{"class":502,"line":532},[500,1806,1807],{"class":506},"    video.",[500,1809,1157],{"class":510},[500,1811,1812],{"class":506},"(cachedConfig)\n",[500,1814,1815,1818,1820],{"class":502,"line":540},[500,1816,1817],{"class":506},"    decoderNeedsReset ",[500,1819,699],{"class":681},[500,1821,1822],{"class":572}," false\n",[500,1824,1825],{"class":502,"line":552},[500,1826,614],{"class":506},[137,1828,1830],{"id":1829},"backpressure-and-timestamp-deduplication","Backpressure and Timestamp Deduplication",[114,1832,1833,1834,1837,1838,1841],{},"If ",[133,1835,1836],{},"decoder.decodeQueueSize > 5",", incoming P-frames are dropped rather than queued — the threshold of 5 (rather than 2) tolerates hardware decoder startup latency without causing unnecessary stutter. Separately, after rendering, ",[133,1839,1840],{},"lastRenderedPts"," is recorded; a frame whose timestamp is older (out-of-order arrival) is dropped unless it's a keyframe:",[126,1843,1845],{"className":1032,"code":1844,"language":1034,"meta":135,"style":135},"if (packet.timestamp \u003C this.lastRenderedPts && !packet.isKeyFrame) {\n    return\n}\n",[133,1846,1847,1868,1873],{"__ignoreMap":135},[500,1848,1849,1851,1854,1857,1859,1862,1864,1866],{"class":502,"line":503},[500,1850,1554],{"class":681},[500,1852,1853],{"class":506}," (packet.timestamp ",[500,1855,1856],{"class":681},"\u003C",[500,1858,1077],{"class":572},[500,1860,1861],{"class":506},".lastRenderedPts ",[500,1863,1567],{"class":681},[500,1865,1796],{"class":681},[500,1867,1613],{"class":506},[500,1869,1870],{"class":502,"line":523},[500,1871,1872],{"class":681},"    return\n",[500,1874,1875],{"class":502,"line":532},[500,1876,614],{"class":506},[29,1878,79],{"id":1879},"orientation-change-handling",[137,1881,1883],{"id":1882},"encoder-rebuild","Encoder Rebuild",[114,1885,1886,1887,1890,1891,1894,1895,1898,1899,1902,1903,1906],{},"An ",[133,1888,1889],{},"OrientationEventListener"," in ",[133,1892,1893],{},"ScreenMirrorService"," compares the display's ",[133,1896,1897],{},"rotation"," against the cached ",[133,1900,1901],{},"isPortrait"," flag on every sensor callback; only a genuine portrait\u002Flandscape flip calls ",[133,1904,1905],{},"pipeline.onOrientationChanged()"," and invalidates the accessibility screen-size cache used for touch coordinate scaling.",[114,1908,1909],{},[180,1910],{"alt":1911,"src":1912},"Diagram 4","\u002Fblog\u002Fscreen-mirror\u002Fdiagram-04.svg",[114,1914,1915,671],{},[133,1916,1917],{},"rebuildEncoderAndResize()",[1723,1919,1920,1923,1932,1935],{},[37,1921,1922],{},"Create a new encoder at the new dimensions (e.g. landscape 1920×1080)",[37,1924,1925,1926,1929,1930],{},"Switch ",[133,1927,1928],{},"VirtualDisplay.surface"," to the new encoder's input ",[133,1931,627],{},[37,1933,1934],{},"Stop the old encoder",[37,1936,1937,1940],{},[133,1938,1939],{},"VirtualDisplay.resize()"," to the new dimensions",[114,1942,1943,1944,1947,1948,1951],{},"Surface switch happens ",[118,1945,1946],{},"before"," resize — ensuring the new encoder receives frames first, and the old encoder is stopped before it can receive wrong-dimension frames. If ",[133,1949,1950],{},"virtualDisplay?.surface = ..."," throws, the rebuild aborts and keeps the old encoder running rather than leaving the pipeline with no encoder at all.",[137,1953,1955],{"id":1954},"config-change-notification","Config Change Notification",[114,1957,1958,1959,1962,1963,1965],{},"When the new encoder first outputs SPS\u002FPPS, ",[133,1960,1961],{},"pendingConfigBroadcast"," is set on the pipeline. When the first IDR from the new encoder arrives, it's bundled with that config into a single ",[133,1964,777],{}," event instead of being sent as an ordinary video packet.",[114,1967,1968,1969,671],{},"The web client then, in ",[133,1970,1971],{},"handleConfig()",[1723,1973,1974,1977,1980,1986],{},[37,1975,1976],{},"Reconfigures the decoder with the new SPS\u002FPPS",[37,1978,1979],{},"Decodes the bundled IDR frame immediately",[37,1981,1982,1983,1985],{},"Calls ",[133,1984,1260],{}," to drop any residual P-frames from the old encoder still in flight",[37,1987,1982,1988,1990],{},[133,1989,1268],{}," to request a clean, fresh IDR",[114,1992,1993,1994,1996],{},"Steps 3-4 are a safety net — even if the new encoder's first IDR has incorrect dimensions (during the async resize window), the web client quickly recovers to the correct dimensions. ",[133,1995,1971],{}," also short-circuits if the incoming config is byte-identical to the cached one, since reconfiguring the decoder with unchanged bytes is a no-op that still costs an IDR to recover from.",[29,1998,85],{"id":1999},"system-mediaprojection-lifecycle",[137,2001,2003],{"id":2002},"the-problem","The Problem",[114,2005,2006,2007,2009,2010,2013,2014,2016,2017,2020,2021,2023],{},"Users may close the system-level screen cast (MediaProjection) via the Android system notification bar, rather than through the app's UI. In this case, ",[133,2008,1893],{}," doesn't know casting has stopped — ",[133,2011,2012],{},"running"," remains ",[133,2015,1189],{},", the web client queries ",[133,2018,2019],{},"screenMirrorState"," and gets ",[133,2022,1189],{},", but no video frames arrive, and the page is stuck on loading.",[137,2025,2027],{"id":2026},"mediaprojectioncallback","MediaProjection.Callback",[114,2029,2030,2032,2033,2036,2037,2040,2041,1890,2044,671],{},[133,2031,216],{}," provides a ",[133,2034,2035],{},"Callback.onStop()"," callback that fires when the system stops casting. ",[133,2038,2039],{},"ScreenMirrorPipeline.startEncoders()"," registers this callback and calls ",[133,2042,2043],{},"ScreenMirrorService.instance?.stop()",[133,2045,2046],{},"onStop()",[126,2048,2050],{"className":494,"code":2049,"language":496,"meta":135,"style":135},"projection.registerCallback(object : MediaProjection.Callback() {\n    override fun onStop() {\n        ScreenMirrorService.instance?.stop()\n    }\n}, null)\n",[133,2051,2052,2077,2090,2101,2105],{"__ignoreMap":135},[500,2053,2054,2057,2060,2062,2065,2068,2070,2072,2075],{"class":502,"line":503},[500,2055,2056],{"class":506},"projection.",[500,2058,2059],{"class":510},"registerCallback",[500,2061,1390],{"class":506},[500,2063,2064],{"class":681},"object",[500,2066,2067],{"class":506}," : ",[500,2069,216],{"class":510},[500,2071,1080],{"class":506},[500,2073,2074],{"class":510},"Callback",[500,2076,688],{"class":506},[500,2078,2079,2082,2085,2088],{"class":502,"line":523},[500,2080,2081],{"class":681},"    override",[500,2083,2084],{"class":681}," fun",[500,2086,2087],{"class":510}," onStop",[500,2089,688],{"class":506},[500,2091,2092,2095,2098],{"class":502,"line":532},[500,2093,2094],{"class":506},"        ScreenMirrorService.instance?.",[500,2096,2097],{"class":510},"stop",[500,2099,2100],{"class":506},"()\n",[500,2102,2103],{"class":502,"line":540},[500,2104,1659],{"class":506},[500,2106,2107,2110,2113],{"class":502,"line":552},[500,2108,2109],{"class":506},"}, ",[500,2111,2112],{"class":572},"null",[500,2114,576],{"class":506},[114,2116,2117],{},[180,2118],{"alt":2119,"src":2120},"Diagram 5","\u002Fblog\u002Fscreen-mirror\u002Fdiagram-05.svg",[137,2122,2124],{"id":2123},"servicestop-responsibilities","Service.stop() Responsibilities",[114,2126,2127,2130],{},[133,2128,2129],{},"stop()"," is the explicit stop point, responsible for notifying the web client and stopping the service:",[126,2132,2134],{"className":494,"code":2133,"language":496,"meta":135,"style":135},"fun stop() {\n    if (!running) return  \u002F\u002F prevent recursion\n    running = false\n    sendEvent(WebSocketEvent(EventType.SCREEN_MIRRORING, \"\"\"{\"running\":false}\"\"\"))\n    stopForeground(STOP_FOREGROUND_REMOVE)\n    stopSelf()\n}\n",[133,2135,2136,2145,2162,2171,2190,2198,2205],{"__ignoreMap":135},[500,2137,2138,2140,2143],{"class":502,"line":503},[500,2139,682],{"class":681},[500,2141,2142],{"class":510}," stop",[500,2144,688],{"class":506},[500,2146,2147,2149,2151,2153,2156,2159],{"class":502,"line":523},[500,2148,1606],{"class":681},[500,2150,325],{"class":506},[500,2152,1559],{"class":681},[500,2154,2155],{"class":506},"running) ",[500,2157,2158],{"class":681},"return",[500,2160,2161],{"class":548},"  \u002F\u002F prevent recursion\n",[500,2163,2164,2167,2169],{"class":502,"line":532},[500,2165,2166],{"class":506},"    running ",[500,2168,699],{"class":681},[500,2170,1822],{"class":572},[500,2172,2173,2176,2178,2181,2184,2187],{"class":502,"line":540},[500,2174,2175],{"class":510},"    sendEvent",[500,2177,1390],{"class":506},[500,2179,2180],{"class":510},"WebSocketEvent",[500,2182,2183],{"class":506},"(EventType.SCREEN_MIRRORING, ",[500,2185,2186],{"class":1176},"\"\"\"{\"running\":false}\"\"\"",[500,2188,2189],{"class":506},"))\n",[500,2191,2192,2195],{"class":502,"line":552},[500,2193,2194],{"class":510},"    stopForeground",[500,2196,2197],{"class":506},"(STOP_FOREGROUND_REMOVE)\n",[500,2199,2200,2203],{"class":502,"line":563},[500,2201,2202],{"class":510},"    stopSelf",[500,2204,2100],{"class":506},[500,2206,2207],{"class":502,"line":579},[500,2208,614],{"class":506},[114,2210,1307,2211,2214,2215,284,2217,284,2219,284,2222,284,2225,284,2228,284,2231,284,2233,2235,2236,2239],{},[133,2212,2213],{},"if (!running) return"," guard prevents recursion: ",[133,2216,2046],{},[133,2218,2129],{},[133,2220,2221],{},"stopSelf()",[133,2223,2224],{},"onDestroy()",[133,2226,2227],{},"pipeline.stop()",[133,2229,2230],{},"projection.stop()",[133,2232,2046],{},[133,2234,2129],{}," (at this point ",[133,2237,2238],{},"running=false",", returns immediately).",[137,2241,2243],{"id":2242},"web-side-handling","Web-Side Handling",[114,2245,2246,2247,2250],{},"When the web client receives the ",[133,2248,2249],{},"{\"running\":false}"," event, it resets to idle state and shows the start button:",[126,2252,2254],{"className":1032,"code":2253,"language":1034,"meta":135,"style":135},"const onScreenMirroring = (data: any) => {\n    if (data?.running === false) {\n        cleanupFn()\n        fullReset()\n        return\n    }\n    \u002F\u002F running=true → connect to stream\n}\n",[133,2255,2256,2281,2296,2303,2310,2314,2318,2323],{"__ignoreMap":135},[500,2257,2258,2260,2263,2265,2267,2270,2272,2275,2277,2279],{"class":502,"line":503},[500,2259,1041],{"class":681},[500,2261,2262],{"class":510}," onScreenMirroring",[500,2264,1047],{"class":681},[500,2266,325],{"class":506},[500,2268,2269],{"class":1067},"data",[500,2271,671],{"class":681},[500,2273,2274],{"class":572}," any",[500,2276,1071],{"class":506},[500,2278,1074],{"class":681},[500,2280,520],{"class":506},[500,2282,2283,2285,2288,2291,2294],{"class":502,"line":523},[500,2284,1606],{"class":681},[500,2286,2287],{"class":506}," (data?.running ",[500,2289,2290],{"class":681},"===",[500,2292,2293],{"class":572}," false",[500,2295,1601],{"class":506},[500,2297,2298,2301],{"class":502,"line":532},[500,2299,2300],{"class":510},"        cleanupFn",[500,2302,2100],{"class":506},[500,2304,2305,2308],{"class":502,"line":540},[500,2306,2307],{"class":510},"        fullReset",[500,2309,2100],{"class":506},[500,2311,2312],{"class":502,"line":552},[500,2313,1654],{"class":681},[500,2315,2316],{"class":502,"line":563},[500,2317,1659],{"class":506},[500,2319,2320],{"class":502,"line":579},[500,2321,2322],{"class":548},"    \u002F\u002F running=true → connect to stream\n",[500,2324,2325],{"class":502,"line":587},[500,2326,614],{"class":506},[29,2328,91],{"id":2329},"remote-control-touch-injection",[114,2331,2332,2333,1080],{},"Screen mirroring is one-way by default (video\u002Faudio only); remote control is opt-in and requires the user to enable PlainApp's Accessibility Service once, since Android has no public API for injecting arbitrary touch events outside of ",[133,2334,2335],{},"AccessibilityService.dispatchGesture()",[114,2337,2338],{},[180,2339],{"alt":2340,"src":2341},"Diagram 6","\u002Fblog\u002Fscreen-mirror\u002Fdiagram-06.svg",[137,2343,2345],{"id":2344},"coordinate-normalization-web","Coordinate Normalization (Web)",[114,2347,2348,2349,2352,2353,2356,2357,2360,2361,2364,2365,2368],{},"A transparent overlay sits above the ",[133,2350,2351],{},"\u003Ccanvas>"," and captures pointer events. ",[133,2354,2355],{},"normalizeCoords()"," converts a raw ",[133,2358,2359],{},"clientX\u002FclientY"," into ",[133,2362,2363],{},"[0,1]"," coordinates relative to the ",[790,2366,2367],{},"actual video content area"," — not the overlay's bounding box — by computing the letterbox\u002Fpillarbox offset from the canvas's backing-store aspect ratio vs. its rendered container aspect ratio:",[126,2370,2372],{"className":1032,"code":2371,"language":1034,"meta":135,"style":135},"if (videoAspect > containerAspect) {\n    \u002F\u002F Letterboxed top\u002Fbottom\n    renderW = containerW\n    renderH = containerW \u002F videoAspect\n    offsetY = (containerH - renderH) \u002F 2\n} else {\n    \u002F\u002F Pillarboxed left\u002Fright\n    renderH = containerH\n    renderW = containerH * videoAspect\n    offsetX = (containerW - renderW) \u002F 2\n}\n",[133,2373,2374,2386,2391,2401,2416,2437,2447,2452,2461,2475,2494],{"__ignoreMap":135},[500,2375,2376,2378,2381,2383],{"class":502,"line":503},[500,2377,1554],{"class":681},[500,2379,2380],{"class":506}," (videoAspect ",[500,2382,1575],{"class":681},[500,2384,2385],{"class":506}," containerAspect) {\n",[500,2387,2388],{"class":502,"line":523},[500,2389,2390],{"class":548},"    \u002F\u002F Letterboxed top\u002Fbottom\n",[500,2392,2393,2396,2398],{"class":502,"line":532},[500,2394,2395],{"class":506},"    renderW ",[500,2397,699],{"class":681},[500,2399,2400],{"class":506}," containerW\n",[500,2402,2403,2406,2408,2411,2413],{"class":502,"line":540},[500,2404,2405],{"class":506},"    renderH ",[500,2407,699],{"class":681},[500,2409,2410],{"class":506}," containerW ",[500,2412,751],{"class":681},[500,2414,2415],{"class":506}," videoAspect\n",[500,2417,2418,2421,2423,2426,2429,2432,2434],{"class":502,"line":552},[500,2419,2420],{"class":506},"    offsetY ",[500,2422,699],{"class":681},[500,2424,2425],{"class":506}," (containerH ",[500,2427,2428],{"class":681},"-",[500,2430,2431],{"class":506}," renderH) ",[500,2433,751],{"class":681},[500,2435,2436],{"class":572}," 2\n",[500,2438,2439,2442,2445],{"class":502,"line":563},[500,2440,2441],{"class":506},"} ",[500,2443,2444],{"class":681},"else",[500,2446,520],{"class":506},[500,2448,2449],{"class":502,"line":579},[500,2450,2451],{"class":548},"    \u002F\u002F Pillarboxed left\u002Fright\n",[500,2453,2454,2456,2458],{"class":502,"line":587},[500,2455,2405],{"class":506},[500,2457,699],{"class":681},[500,2459,2460],{"class":506}," containerH\n",[500,2462,2463,2465,2467,2470,2473],{"class":502,"line":599},[500,2464,2395],{"class":506},[500,2466,699],{"class":681},[500,2468,2469],{"class":506}," containerH ",[500,2471,2472],{"class":681},"*",[500,2474,2415],{"class":506},[500,2476,2477,2480,2482,2485,2487,2490,2492],{"class":502,"line":611},[500,2478,2479],{"class":506},"    offsetX ",[500,2481,699],{"class":681},[500,2483,2484],{"class":506}," (containerW ",[500,2486,2428],{"class":681},[500,2488,2489],{"class":506}," renderW) ",[500,2491,751],{"class":681},[500,2493,2436],{"class":572},[500,2495,2496],{"class":502,"line":1170},[500,2497,614],{"class":506},[114,2499,2500,2501,2504,2505,2508,2509,2512,2513,2516],{},"A pointer press starts a ",[133,2502,2503],{},"GestureState"," that tracks start position\u002Ftime; a 500ms hold with \u003C 10px movement escalates to ",[133,2506,2507],{},"LONG_PRESS",", movement past that threshold becomes a ",[133,2510,2511],{},"SWIPE",", and a quick release is a ",[133,2514,2515],{},"TAP",". A visual touch indicator (a growing\u002Ffading dot) gives the operator feedback on what gesture was recognized, before the phone even responds.",[137,2518,2520],{"id":2519},"graphql-accessibilityservice","GraphQL → AccessibilityService",[114,2522,2523,2524,2527,2528,325,2531,751,2533,751,2535,751,2537,751,2540,751,2543,751,2546,751,2549,751,2552,2555,2556,2559,2560,2563,2564,671],{},"Every recognized gesture is sent as one ",[133,2525,2526],{},"sendScreenMirrorControl(input)"," mutation carrying an ",[133,2529,2530],{},"action",[133,2532,2515],{},[133,2534,2507],{},[133,2536,2511],{},[133,2538,2539],{},"SCROLL",[133,2541,2542],{},"BACK",[133,2544,2545],{},"HOME",[133,2547,2548],{},"RECENTS",[133,2550,2551],{},"LOCK_SCREEN",[133,2553,2554],{},"KEY",") plus normalized coordinates. The resolver calls ",[133,2557,2558],{},"dispatchScreenMirrorControl()",", which multiplies the normalized coordinates by the real screen size (from ",[133,2561,2562],{},"PlainAccessibilityService.getScreenSize()",", invalidated on every orientation change) and delegates to ",[133,2565,2566],{},"PlainAccessibilityService.dispatchControl()",[126,2568,2570],{"className":494,"code":2569,"language":496,"meta":135,"style":135},"private fun dispatchTap(x: Float, y: Float) {\n    val path = Path().apply { moveTo(x, y) }\n    val stroke = GestureDescription.StrokeDescription(path, 0, 50)\n    dispatchGesture(GestureDescription.Builder().addStroke(stroke).build(), null, null)\n}\n",[133,2571,2572,2595,2619,2646,2679],{"__ignoreMap":135},[500,2573,2574,2577,2579,2582,2585,2588,2591,2593],{"class":502,"line":503},[500,2575,2576],{"class":681},"private",[500,2578,2084],{"class":681},[500,2580,2581],{"class":510}," dispatchTap",[500,2583,2584],{"class":506},"(x: ",[500,2586,2587],{"class":510},"Float",[500,2589,2590],{"class":506},", y: ",[500,2592,2587],{"class":510},[500,2594,1601],{"class":506},[500,2596,2597,2599,2602,2604,2607,2609,2611,2613,2616],{"class":502,"line":523},[500,2598,693],{"class":681},[500,2600,2601],{"class":506}," path ",[500,2603,699],{"class":681},[500,2605,2606],{"class":510}," Path",[500,2608,705],{"class":506},[500,2610,517],{"class":510},[500,2612,710],{"class":506},[500,2614,2615],{"class":510},"moveTo",[500,2617,2618],{"class":506},"(x, y) }\n",[500,2620,2621,2623,2626,2628,2631,2634,2637,2639,2641,2644],{"class":502,"line":532},[500,2622,693],{"class":681},[500,2624,2625],{"class":506}," stroke ",[500,2627,699],{"class":681},[500,2629,2630],{"class":506}," GestureDescription.",[500,2632,2633],{"class":510},"StrokeDescription",[500,2635,2636],{"class":506},"(path, ",[500,2638,413],{"class":572},[500,2640,1429],{"class":506},[500,2642,2643],{"class":572},"50",[500,2645,576],{"class":506},[500,2647,2648,2651,2654,2657,2659,2662,2665,2668,2671,2673,2675,2677],{"class":502,"line":540},[500,2649,2650],{"class":510},"    dispatchGesture",[500,2652,2653],{"class":506},"(GestureDescription.",[500,2655,2656],{"class":510},"Builder",[500,2658,705],{"class":506},[500,2660,2661],{"class":510},"addStroke",[500,2663,2664],{"class":506},"(stroke).",[500,2666,2667],{"class":510},"build",[500,2669,2670],{"class":506},"(), ",[500,2672,2112],{"class":572},[500,2674,1429],{"class":506},[500,2676,2112],{"class":572},[500,2678,576],{"class":506},[500,2680,2681],{"class":502,"line":552},[500,2682,614],{"class":506},[114,2684,2685,620,2687,2689,2690,2693,2694,2696,2697,2700,2701,2704,2705,751,2707,751,2709,751,2711,2713,2714,2717,2718,2721],{},[133,2686,2511],{},[133,2688,2507],{}," build the same ",[133,2691,2692],{},"GestureDescription"," with a longer stroke duration or a line path instead of a single point; ",[133,2695,2539],{}," is implemented as a synthetic swipe from ",[133,2698,2699],{},"(x, y)"," to ",[133,2702,2703],{},"(x, y + deltaY)"," clamped to ±500px. The four global actions (",[133,2706,2542],{},[133,2708,2545],{},[133,2710,2548],{},[133,2712,2551],{},") skip gesture dispatch entirely and call ",[133,2715,2716],{},"performGlobalAction()"," directly. If the Accessibility Service isn't enabled, the resolver throws a ",[133,2719,2720],{},"GraphQLError"," rather than silently dropping the input, so the web UI can prompt the user to enable it.",[29,2723,97],{"id":2724},"audio-pipeline",[137,2726,2728],{"id":2727},"android-opus-encoding","Android Opus Encoding",[114,2730,2731,2734,2735,2738,2739,2741,2742,2745,2746,2748,2749,2752,2753,2756,2757,2759,2760,2763,2764,2767],{},[133,2732,2733],{},"MediaCodecAudioEncoder"," uses ",[133,2736,2737],{},"AudioPlaybackCaptureConfiguration"," (built from the same ",[133,2740,216],{},") to capture system audio via ",[133,2743,2744],{},"AudioRecord",", feeding raw PCM into a ",[133,2747,233],{}," Opus encoder. This requires Android 10+ and the ",[133,2750,2751],{},"RECORD_AUDIO"," permission — on older devices or without the permission, ",[133,2754,2755],{},"start()"," logs a warning and skips audio entirely (video keeps working). Encoded Opus packets are wrapped in the same ",[133,2758,806],{}," protocol (with ",[133,2761,2762],{},"FLAG_AUDIO"," set) and share the video packet's ",[133,2765,2766],{},"SCREEN_MIRROR_AUDIO"," WebSocket channel.",[137,2769,2771],{"id":2770},"web-opus-decoding","Web Opus Decoding",[114,2773,2774,2734,2777,2779,2780,2783,2784,2786,2787,2790],{},[133,2775,2776],{},"ScreenMirrorAudioPipeline",[133,2778,283],{}," to decode Opus data, outputting ",[133,2781,2782],{},"AudioData"," routed to an ",[133,2785,287],{}," element. Audio frame ",[133,2788,2789],{},"timestamp"," is used for A\u002FV sync — sharing the same time base (encoder PTS, in microseconds) as video frames, so no separate clock negotiation is needed between the two streams.",[29,2792,103],{"id":2793},"performance-optimizations",[137,2795,2797],{"id":2796},"zero-copy-paths","Zero-Copy Paths",[185,2799,2800,2810],{},[188,2801,2802],{},[191,2803,2804,2807],{},[194,2805,2806],{},"Path",[194,2808,2809],{},"Method",[204,2811,2812,2820,2831],{},[191,2813,2814,2817],{},[209,2815,2816],{},"VirtualDisplay → encoder Surface",[209,2818,2819],{},"GPU direct, Surface passthrough",[191,2821,2822,2825],{},[209,2823,2824],{},"VideoDecoder → VideoFrame → WebGL texture",[209,2826,2827,2830],{},[133,2828,2829],{},"gl.texImage2D(VideoFrame)",", GPU direct",[191,2832,2833,2836],{},[209,2834,2835],{},"WebSocket receive → VideoPacket parse",[209,2837,2838,2840],{},[133,2839,995],{}," is a view, no copy",[137,2842,2844],{"id":2843},"avcctoannexb-optimization","avccToAnnexB Optimization",[114,2846,2847,2848,2851],{},"Some Android encoders output AVCC format (4-byte length prefix), which needs conversion to Annex-B format (",[133,2849,2850],{},"00 00 00 01"," start code) for WebCodecs decoding.",[114,2853,2854,2855,2858,2859,2862,2863,2866,2867,2870],{},"The early implementation used ",[133,2856,2857],{},"ArrayList\u003CByte>"," with per-byte boxing — a 50KB IDR frame produced 50,000 ",[133,2860,2861],{},"java.lang.Byte"," boxing operations, creating massive GC pressure. The optimization uses two-pass scan + ",[133,2864,2865],{},"copyInto"," (which maps to the ",[133,2868,2869],{},"System.arraycopy"," intrinsic on JVM):",[126,2872,2874],{"className":494,"code":2873,"language":496,"meta":135,"style":135},"\u002F\u002F First pass: compute output size\nvar outSize = 0\n\u002F\u002F Second pass: bulk copy\nval out = ByteArray(outSize)\navcc.copyInto(out, writeOff + 4, off + 4, off + 4 + len)\n",[133,2875,2876,2881,2893,2898,2914],{"__ignoreMap":135},[500,2877,2878],{"class":502,"line":503},[500,2879,2880],{"class":548},"\u002F\u002F First pass: compute output size\n",[500,2882,2883,2886,2889,2891],{"class":502,"line":523},[500,2884,2885],{"class":681},"var",[500,2887,2888],{"class":506}," outSize ",[500,2890,699],{"class":681},[500,2892,1578],{"class":572},[500,2894,2895],{"class":502,"line":532},[500,2896,2897],{"class":548},"\u002F\u002F Second pass: bulk copy\n",[500,2899,2900,2903,2906,2908,2911],{"class":502,"line":540},[500,2901,2902],{"class":681},"val",[500,2904,2905],{"class":506}," out ",[500,2907,699],{"class":681},[500,2909,2910],{"class":510}," ByteArray",[500,2912,2913],{"class":506},"(outSize)\n",[500,2915,2916,2919,2921,2924,2926,2929,2932,2934,2936,2938,2940,2942,2945],{"class":502,"line":552},[500,2917,2918],{"class":506},"avcc.",[500,2920,2865],{"class":510},[500,2922,2923],{"class":506},"(out, writeOff ",[500,2925,1595],{"class":681},[500,2927,2928],{"class":572}," 4",[500,2930,2931],{"class":506},", off ",[500,2933,1595],{"class":681},[500,2935,2928],{"class":572},[500,2937,2931],{"class":506},[500,2939,1595],{"class":681},[500,2941,2928],{"class":572},[500,2943,2944],{"class":681}," +",[500,2946,2947],{"class":506}," len)\n",[137,2949,2951],{"id":2950},"p-frame-drop-strategy","P-Frame Drop Strategy",[114,2953,2954],{},"The decoder may be slow during initialization. If the P-frame queue is too long, latency accumulates. The decode queue size threshold is set to > 5 (rather than > 2) to avoid excessive frame loss during hardware decoder initialization.",[137,2956,2958],{"id":2957},"idr-request-deduplication","IDR Request Deduplication",[114,2960,1307,2961,2963],{},[133,2962,1672],{}," guard ensures only one IDR request per loss event, preventing duplicate requests while waiting for an IDR to arrive.",[29,2965,109],{"id":2966},"design-patterns-recap",[185,2968,2969,2982],{},[188,2970,2971],{},[191,2972,2973,2976,2979],{},[194,2974,2975],{},"Pattern",[194,2977,2978],{},"Where",[194,2980,2981],{},"Why",[204,2983,2984,2999,3019,3034,3049,3062,3083,3100,3115,3129,3143],{},[191,2985,2986,2991,2996],{},[209,2987,2988],{},[118,2989,2990],{},"State Machine",[209,2992,2993,2995],{},[133,2994,1672],{}," flag",[209,2997,2998],{},"Explicit P-frame drop\u002Frecovery state transitions",[191,3000,3001,3006,3012],{},[209,3002,3003],{},[118,3004,3005],{},"Recursion Guard",[209,3007,3008,1890,3010],{},[133,3009,2213],{},[133,3011,2129],{},[209,3013,3014,3015,3018],{},"Prevents ",[133,3016,3017],{},"onStop → stop → onDestroy → pipeline.stop → projection.stop → onStop"," recursion",[191,3020,3021,3026,3031],{},[209,3022,3023],{},[118,3024,3025],{},"Zero-Copy Pipeline",[209,3027,3028],{},[133,3029,3030],{},"VideoFrame → gl.texImage2D",[209,3032,3033],{},"GPU direct texture upload, no CPU copy",[191,3035,3036,3041,3046],{},[209,3037,3038],{},[118,3039,3040],{},"Two-Pass Scan",[209,3042,3043],{},[133,3044,3045],{},"avccToAnnexB",[209,3047,3048],{},"Pre-compute size, single allocation + bulk copy, eliminates boxing",[191,3050,3051,3056,3059],{},[209,3052,3053],{},[118,3054,3055],{},"Bundled Event",[209,3057,3058],{},"SPS\u002FPPS + IDR in one event",[209,3060,3061],{},"Config change completes reconfiguration + first-frame decode in one event",[191,3063,3064,3069,3080],{},[209,3065,3066],{},[118,3067,3068],{},"Callback Separation",[209,3070,3071,3073,3074,3073,3077],{},[133,3072,1310],{}," vs ",[133,3075,3076],{},"onDisconnected",[133,3078,3079],{},"onScreenMirrorOff",[209,3081,3082],{},"Clear distinction between first-frame render, transport failure, and phone-side stop",[191,3084,3085,3090,3097],{},[209,3086,3087],{},[118,3088,3089],{},"Safety Net",[209,3091,3092,217,3095],{},[133,3093,3094],{},"requestIdr()",[133,3096,1268],{},[209,3098,3099],{},"Drop residual frames + request clean IDR after config change",[191,3101,3102,3107,3112],{},[209,3103,3104],{},[118,3105,3106],{},"PTS Deduplication",[209,3108,3109],{},[133,3110,3111],{},"timestamp \u003C lastRenderedPts",[209,3113,3114],{},"Drop out-of-order frames",[191,3116,3117,3122,3126],{},[209,3118,3119],{},[118,3120,3121],{},"FrameId Gap",[209,3123,3124],{},[133,3125,1543],{},[209,3127,3128],{},"ACK-free packet loss detection",[191,3130,3131,3135,3140],{},[209,3132,3133],{},[118,3134,1367],{},[209,3136,3137,3138],{},"WebGL2 ",[133,3139,1487],{},[209,3141,3142],{},"Bypass compositor, save 1 frame of latency",[191,3144,3145,3150,3157],{},[209,3146,3147],{},[118,3148,3149],{},"Explicit Fail Fast",[209,3151,3152,3154,3155],{},[133,3153,328],{}," throws ",[133,3156,2720],{},[209,3158,3159],{},"Surfaces \"accessibility disabled\" instead of silently dropping input",[29,3161,3163],{"id":3162},"further-reading","Further Reading",[34,3165,3166],{},[37,3167,3168,3173,3174,751,3176,3179],{},[40,3169,1012],{"href":3170,"rel":3171},"https:\u002F\u002Fdeveloper.mozilla.org\u002Fen-US\u002Fdocs\u002FWeb\u002FAPI\u002FWebCodecs_API",[3172],"nofollow"," — MDN documentation covering the ",[133,3175,1016],{},[133,3177,3178],{},"AudioDecoder"," interfaces.",[3181,3182,3183],"style",{},"html pre.shiki code .sVt8B, html code.shiki .sVt8B{--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .sScJk, html code.shiki .sScJk{--shiki-default:#6F42C1;--shiki-dark:#B392F0}html pre.shiki code .sJ8bj, html code.shiki .sJ8bj{--shiki-default:#6A737D;--shiki-dark:#6A737D}html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .szBVR, html code.shiki .szBVR{--shiki-default:#D73A49;--shiki-dark:#F97583}html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}html pre.shiki code .s4XuR, html code.shiki .s4XuR{--shiki-default:#E36209;--shiki-dark:#FFAB70}",{"title":135,"searchDepth":532,"depth":532,"links":3185},[3186,3187,3191,3198,3202,3206,3211,3217,3221,3227,3231,3235,3241,3242],{"id":31,"depth":523,"text":32},{"id":112,"depth":523,"text":43,"children":3188},[3189,3190],{"id":139,"depth":532,"text":140},{"id":175,"depth":532,"text":176},{"id":340,"depth":523,"text":49,"children":3192},[3193,3194,3195,3196,3197],{"id":343,"depth":532,"text":344},{"id":484,"depth":532,"text":485},{"id":642,"depth":532,"text":643},{"id":660,"depth":532,"text":661},{"id":739,"depth":532,"text":740},{"id":800,"depth":523,"text":55,"children":3199},[3200,3201],{"id":810,"depth":532,"text":811},{"id":940,"depth":532,"text":941},{"id":1003,"depth":523,"text":61,"children":3203},[3204,3205],{"id":1006,"depth":532,"text":271},{"id":1241,"depth":532,"text":1242},{"id":1322,"depth":523,"text":67,"children":3207},[3208,3209,3210],{"id":1325,"depth":532,"text":1326},{"id":1366,"depth":532,"text":1367},{"id":1507,"depth":532,"text":1508},{"id":1526,"depth":523,"text":73,"children":3212},[3213,3214,3215,3216],{"id":1529,"depth":532,"text":1530},{"id":1666,"depth":532,"text":1667},{"id":1751,"depth":532,"text":1752},{"id":1829,"depth":532,"text":1830},{"id":1879,"depth":523,"text":79,"children":3218},[3219,3220],{"id":1882,"depth":532,"text":1883},{"id":1954,"depth":532,"text":1955},{"id":1999,"depth":523,"text":85,"children":3222},[3223,3224,3225,3226],{"id":2002,"depth":532,"text":2003},{"id":2026,"depth":532,"text":2027},{"id":2123,"depth":532,"text":2124},{"id":2242,"depth":532,"text":2243},{"id":2329,"depth":523,"text":91,"children":3228},[3229,3230],{"id":2344,"depth":532,"text":2345},{"id":2519,"depth":532,"text":2520},{"id":2724,"depth":523,"text":97,"children":3232},[3233,3234],{"id":2727,"depth":532,"text":2728},{"id":2770,"depth":532,"text":2771},{"id":2793,"depth":523,"text":103,"children":3236},[3237,3238,3239,3240],{"id":2796,"depth":532,"text":2797},{"id":2843,"depth":532,"text":2844},{"id":2950,"depth":532,"text":2951},{"id":2957,"depth":532,"text":2958},{"id":2966,"depth":523,"text":109},{"id":3162,"depth":523,"text":3163},"Architecture","2025-07-29","This article covers the end-to-end design of PlainApp's screen mirror system: how Android captures and hardware-encodes H.264\u002FOpus via MediaCodec, how frames travel over WebSocket using a custom binary protocol, how the web side decodes via WebCodecs and renders via WebGL2 with zero CPU copies, how loss detection, orientation change, and remote touch control are handled, and how the system MediaProjection lifecycle is kept in sync.","md",{},true,"\u002Fblog\u002Fscreen-mirror","15 min read",{"title":24,"description":3245},"How to build ultra-low-latency Android screen mirroring? Detailed guide on MediaCodec hardware encoding, WebCodecs decoding, WebGL2 GPU rendering, packet loss recovery, orientation change handling, and AccessibilityService remote control.","Screen Mirror Architecture: WebCodecs + WebGL2 Low-Latency Casting","blog\u002Fscreen-mirror","oMeBPH88ycaJM9pPH1aGTeiTTdnRa2SpxyUJg4l-UmE",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":3257},"\u003Cpath fill=\"currentColor\" d=\"M224 128a8 8 0 0 1-8 8H59.31l58.35 58.34a8 8 0 0 1-11.32 11.32l-72-72a8 8 0 0 1 0-11.32l72-72a8 8 0 0 1 11.32 11.32L59.31 120H216a8 8 0 0 1 8 8\"\u002F>",1788009009886]