From ccafa34f331be6ea0ed0ed052aab7d828a3ac770 Mon Sep 17 00:00:00 2001 From: Charles Date: Wed, 10 Jan 2024 21:39:06 -0800 Subject: [PATCH] fix: switch to h264 to benefit from hardware --- .gitignore | 1 + cmd/watcher/watcher.go | 44 +++++---- go.mod | 21 +++-- go.sum | 45 +++++++-- internal/h264video/h264video.go | 157 ++++++++++++++++++++++++++++++++ proto/signaler_service.proto | 2 +- sample.yaml | 16 +--- ui/lib/main.dart | 25 ++--- ui/pubspec.lock | 68 +++++++------- 9 files changed, 274 insertions(+), 105 deletions(-) create mode 100644 internal/h264video/h264video.go diff --git a/.gitignore b/.gitignore index 10cdeb2..7437e1a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ server.crt server.key +.vscode diff --git a/cmd/watcher/watcher.go b/cmd/watcher/watcher.go index 233e25d..5bca272 100644 --- a/cmd/watcher/watcher.go +++ b/cmd/watcher/watcher.go @@ -13,8 +13,8 @@ import ( "connectrpc.com/connect" pb "github.com/chathaway-codes/home-sensors/v2/gen" servicepb "github.com/chathaway-codes/home-sensors/v2/gen/genconnect" + "github.com/chathaway-codes/home-sensors/v2/internal/h264video" "github.com/chathaway-codes/home-sensors/v2/internal/sensors" - "github.com/chathaway-codes/home-sensors/v2/internal/video" "github.com/chathaway-codes/home-sensors/v2/internal/watcher/config" "github.com/pion/webrtc/v3" "github.com/pion/webrtc/v3/pkg/media" @@ -59,7 +59,7 @@ func main() { } token := authToken.Msg.GetToken() - vid, err := video.Default.Get() + vid, err := h264video.Default.Get() if err != nil { log.Fatal().Err(err).Msg("failed to get default video") } @@ -111,7 +111,7 @@ func handleSensor(ctx context.Context, client servicepb.SignalerServiceClient, t } } -func handleSession(ctx context.Context, client servicepb.SignalerServiceClient, token string, session *connect.Response[pb.Session], vid *video.Video) { +func handleSession(ctx context.Context, client servicepb.SignalerServiceClient, token string, session *connect.Response[pb.Session], vid *h264video.Video) { var err error log.Debug().Msg("new session") @@ -122,6 +122,25 @@ func handleSession(ctx context.Context, client servicepb.SignalerServiceClient, }, }, }) + if err != nil { + log.Info().Err(err).Msg("failed to get a connection") + return + } + + // connect to the video stream; the cleanup is done in the goroutine which + // consumes the framess + ch, trackCodec, cleanUp := vid.Join() + // Create a video track + videoTrack, videoTrackErr := webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{MimeType: trackCodec}, "video", "pion") + if videoTrackErr != nil { + log.Info().Err(videoTrackErr).Msg("Failed to create video track") + return + } + + rtpSender, err := peerConnection.AddTrack(videoTrack) + if err != nil { + log.Info().Err(err).Msg("Failed to add track to connection") + } // We use the cancel func to signal that the stream is ready iceConnectedCtx, iceConnectedCtxCancel := context.WithCancel(context.Background()) @@ -131,20 +150,6 @@ func handleSession(ctx context.Context, client servicepb.SignalerServiceClient, } }() - // connect to the video stream; the cleanup is done in the goroutine which - // consumes the framess - ch, trackCodec, cleanUp := vid.Join() - // Create a video track - videoTrack, videoTrackErr := webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{MimeType: trackCodec}, "video", "pion") - if videoTrackErr != nil { - log.Info().Err(err).Msg("Failed to create video track") - } - - rtpSender, err := peerConnection.AddTrack(videoTrack) - if err != nil { - log.Info().Err(err).Msg("Failed to add track to connection") - } - // Read incoming RTCP packets // Before these packets are returned they are processed by interceptors. For things // like NACK this needs to be called. @@ -170,8 +175,9 @@ func handleSession(ctx context.Context, client servicepb.SignalerServiceClient, if !readyToSend { continue } - if err := videoTrack.WriteSample(media.Sample{Data: frame, Duration: time.Second}); err != nil { - panic(err) + if err := videoTrack.WriteSample(media.Sample{Data: frame, Duration: time.Millisecond * 33}); err != nil { + log.Err(err).Msg("failed to write sample") + return } } }() diff --git a/go.mod b/go.mod index 987cf1c..6383808 100644 --- a/go.mod +++ b/go.mod @@ -8,11 +8,13 @@ require ( github.com/gofrs/uuid/v5 v5.0.0 github.com/google/go-cmp v0.5.9 github.com/google/uuid v1.3.1 + github.com/pion/rtp v1.8.3 + github.com/pion/rtp/v2 v2.0.0 github.com/pion/webrtc/v3 v3.2.20 + github.com/pion/webrtc/v4 v4.0.0-beta.7 github.com/prometheus/client_golang v1.17.0 github.com/rs/cors v1.10.0 github.com/rs/zerolog v1.31.0 - golang.org/x/net v0.14.0 google.golang.org/grpc v1.58.1 google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v3 v3.0.1 @@ -29,25 +31,24 @@ require ( github.com/pion/datachannel v1.5.5 // indirect github.com/pion/dtls/v2 v2.2.7 // indirect github.com/pion/ice/v2 v2.3.11 // indirect - github.com/pion/interceptor v0.1.18 // indirect + github.com/pion/interceptor v0.1.25 // indirect github.com/pion/logging v0.2.2 // indirect - github.com/pion/mdns v0.0.8 // indirect + github.com/pion/mdns v0.0.9 // indirect github.com/pion/randutil v0.1.0 // indirect - github.com/pion/rtcp v1.2.10 // indirect - github.com/pion/rtp v1.8.1 // indirect - github.com/pion/sctp v1.8.8 // indirect + github.com/pion/rtcp v1.2.12 // indirect + github.com/pion/sctp v1.8.9 // indirect github.com/pion/sdp/v3 v3.0.6 // indirect github.com/pion/srtp/v2 v2.0.17 // indirect github.com/pion/stun v0.6.1 // indirect - github.com/pion/transport/v2 v2.2.3 // indirect + github.com/pion/transport/v2 v2.2.4 // indirect github.com/pion/turn/v2 v2.1.3 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect github.com/prometheus/common v0.44.0 // indirect github.com/prometheus/procfs v0.11.1 // indirect github.com/stretchr/testify v1.8.4 // indirect - golang.org/x/crypto v0.12.0 // indirect - golang.org/x/sys v0.12.0 // indirect - golang.org/x/text v0.12.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/sys v0.13.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect ) diff --git a/go.sum b/go.sum index 85158c1..1026bd1 100644 --- a/go.sum +++ b/go.sum @@ -64,37 +64,54 @@ github.com/pion/dtls/v2 v2.2.7 h1:cSUBsETxepsCSFSxC3mc/aDo14qQLMSL+O6IjG28yV8= github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= github.com/pion/ice/v2 v2.3.11 h1:rZjVmUwyT55cmN8ySMpL7rsS8KYsJERsrxJLLxpKhdw= github.com/pion/ice/v2 v2.3.11/go.mod h1:hPcLC3kxMa+JGRzMHqQzjoSj3xtE9F+eoncmXLlCL4E= -github.com/pion/interceptor v0.1.18 h1:Hk26334NUQeUcJNR27YHYKT+sWNhhegQ9KFz5Nn6yMQ= +github.com/pion/ice/v3 v3.0.2/go.mod h1:q3BDzTsxbqP0ySMSHrFuw2MYGUx/AC3WQfRGC5F/0Is= github.com/pion/interceptor v0.1.18/go.mod h1:tpvvF4cPM6NGxFA1DUMbhabzQBxdWMATDGEUYOR9x6I= +github.com/pion/interceptor v0.1.25 h1:pwY9r7P6ToQ3+IF0bajN0xmk/fNw/suTgaTdlwTDmhc= +github.com/pion/interceptor v0.1.25/go.mod h1:wkbPYAak5zKsfpVDYMtEfWEy8D4zL+rpxCxPImLOg3Y= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= -github.com/pion/mdns v0.0.8 h1:HhicWIg7OX5PVilyBO6plhMetInbzkVJAhbdJiAeVaI= github.com/pion/mdns v0.0.8/go.mod h1:hYE72WX8WDveIhg7fmXgMKivD3Puklk0Ymzog0lSyaI= +github.com/pion/mdns v0.0.9 h1:7Ue5KZsqq8EuqStnpPWV33vYYEH0+skdDN5L7EiEsI4= +github.com/pion/mdns v0.0.9/go.mod h1:2JA5exfxwzXiCihmxpTKgFUpiQws2MnipoPK09vecIc= github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= -github.com/pion/rtcp v1.2.10 h1:nkr3uj+8Sp97zyItdN60tE/S6vk4al5CPRR6Gejsdjc= github.com/pion/rtcp v1.2.10/go.mod h1:ztfEwXZNLGyF1oQDttz/ZKIBaeeg/oWbRYqzBM9TL1I= -github.com/pion/rtp v1.8.1 h1:26OxTc6lKg/qLSGir5agLyj0QKaOv8OP5wps2SFnVNQ= +github.com/pion/rtcp v1.2.12 h1:bKWiX93XKgDZENEXCijvHRU/wRifm6JV5DGcH6twtSM= +github.com/pion/rtcp v1.2.12/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= github.com/pion/rtp v1.8.1/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= +github.com/pion/rtp v1.8.2/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= +github.com/pion/rtp v1.8.3 h1:VEHxqzSVQxCkKDSHro5/4IUUG1ea+MFdqR2R3xSpNU8= +github.com/pion/rtp v1.8.3/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= +github.com/pion/rtp/v2 v2.0.0 h1:8s4xPETm04IugKZaykpJnJ8LAGDLOQpsIpRXMzgM6Ow= +github.com/pion/rtp/v2 v2.0.0/go.mod h1:Vj+rrFbJCT3yxqE/VSwaOo9DQ2pMKGPxuE7hplGOlOs= github.com/pion/sctp v1.8.5/go.mod h1:SUFFfDpViyKejTAdwD1d/HQsCu+V/40cCs2nZIvC3s0= -github.com/pion/sctp v1.8.8 h1:5EdnnKI4gpyR1a1TwbiS/wxEgcUWBHsc7ILAjARJB+U= github.com/pion/sctp v1.8.8/go.mod h1:igF9nZBrjh5AtmKc7U30jXltsFHicFCXSmWA2GWRaWs= +github.com/pion/sctp v1.8.9 h1:TP5ZVxV5J7rz7uZmbyvnUvsn7EJ2x/5q9uhsTtXbI3g= +github.com/pion/sctp v1.8.9/go.mod h1:cMLT45jqw3+jiJCrtHVwfQLnfR0MGZ4rgOJwUOIqLkI= github.com/pion/sdp/v3 v3.0.6 h1:WuDLhtuFUUVpTfus9ILC4HRyHsW6TdugjEX/QY9OiUw= github.com/pion/sdp/v3 v3.0.6/go.mod h1:iiFWFpQO8Fy3S5ldclBkpXqmWy02ns78NOKoLLL0YQw= github.com/pion/srtp/v2 v2.0.17 h1:ECuOk+7uIpY6HUlTb0nXhfvu4REG2hjtC4ronYFCZE4= github.com/pion/srtp/v2 v2.0.17/go.mod h1:y5WSHcJY4YfNB/5r7ca5YjHeIr1H3LM1rKArGGs8jMc= +github.com/pion/srtp/v3 v3.0.1/go.mod h1:3R3a1qIOIxBkVTLGFjafKK6/fJoTdQDhcC67HOyMbJ8= github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4= github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/8= +github.com/pion/stun/v2 v2.0.0/go.mod h1:22qRSh08fSEttYUmJZGlriq9+03jtVmXNODgLccj8GQ= github.com/pion/transport v0.14.1 h1:XSM6olwW+o8J4SCmOBb/BpwZypkHeyM0PGFCxNQBr40= github.com/pion/transport v0.14.1/go.mod h1:4tGmbk00NeYA3rUa9+n+dzCCoKkcy3YlYb99Jn2fNnI= github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= github.com/pion/transport/v2 v2.2.2/go.mod h1:OJg3ojoBJopjEeECq2yJdXH9YVrUJ1uQ++NjXLOUorc= -github.com/pion/transport/v2 v2.2.3 h1:XcOE3/x41HOSKbl1BfyY1TF1dERx7lVvlMCbXU7kfvA= github.com/pion/transport/v2 v2.2.3/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= +github.com/pion/transport/v2 v2.2.4 h1:41JJK6DZQYSeVLxILA2+F4ZkKb4Xd/tFJZRFZQ9QAlo= +github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= +github.com/pion/transport/v3 v3.0.1 h1:gDTlPJwROfSfz6QfSi0ZmeCSkFcnWWiiR9ES0ouANiM= +github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0= github.com/pion/turn/v2 v2.1.3 h1:pYxTVWG2gpC97opdRc5IGsQ1lJ9O/IlNhkzj7MMrGAA= github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= +github.com/pion/turn/v3 v3.0.1/go.mod h1:MrJDKgqryDyWy1/4NT9TWfXWGMC7UHT6pJIv1+gMeNE= github.com/pion/webrtc/v3 v3.2.20 h1:BQJiXQsJq9LgLp3op7rLy1y8d2WD+LtiS9cpY0uQ22A= github.com/pion/webrtc/v3 v3.2.20/go.mod h1:vVURQTBOG5BpWKOJz3nlr23NfTDeyKVmubRNqzQp+Tg= +github.com/pion/webrtc/v4 v4.0.0-beta.7 h1:OGCl69njLUKzT0ozJEon18W1LqH0GtuxG9Qx+qtxBdg= +github.com/pion/webrtc/v4 v4.0.0-beta.7/go.mod h1:/zWz+1e1qrjaIKYZG/mOfPrntiHOhnd3vGz2Fdo85Ys= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -133,8 +150,10 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -152,8 +171,10 @@ golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -184,8 +205,9 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -195,6 +217,8 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -204,8 +228,9 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= diff --git a/internal/h264video/h264video.go b/internal/h264video/h264video.go new file mode 100644 index 0000000..6c582f6 --- /dev/null +++ b/internal/h264video/h264video.go @@ -0,0 +1,157 @@ +// Package h264video implements logic to stream video from a config. +package h264video + +import ( + "bufio" + "context" + "errors" + "io" + "os/exec" + "sync" + "time" + + "github.com/chathaway-codes/home-sensors/v2/internal/pipespy" + "github.com/chathaway-codes/home-sensors/v2/internal/watcher/config" + "github.com/google/uuid" + "github.com/pion/webrtc/v3" + "github.com/pion/webrtc/v4/pkg/media/h264reader" + "github.com/rs/zerolog/log" +) + +var Default = &Mod{} + +type Video struct { + mu sync.Mutex + h264Cmd *exec.Cmd + ivfListeners map[string]chan<- []byte + ivfCodecReady chan struct{} + cancelFunc func() + ctx context.Context +} + +func New(cfg *config.Config) (*Video, error) { + ctx, cancelFunc := context.WithCancel(context.Background()) + // Setup commands + h264cmd := exec.CommandContext(ctx, cfg.H264Cmd.Binary, cfg.H264Cmd.Arguments...) + + return &Video{ + h264Cmd: h264cmd, + ivfListeners: make(map[string]chan<- []byte), + ivfCodecReady: make(chan struct{}), + cancelFunc: cancelFunc, + ctx: ctx, + }, nil +} + +// Run launches the commands and begins creating data. It will block until Done is called. +func (v *Video) Run() { + pipe := pipespy.New() + + h264snoop := pipe.Add(pipespy.NewCmd(v.h264Cmd)).Snoop() + + log.Info().Str("cmd", v.h264Cmd.String()).Msg("h264 command") + + // Log stderr if it appears + go logToStdErr("h264", &v.h264Cmd.Stderr) + + cleanUp := pipe.Start() + defer func() { + errs := cleanUp() + for _, err := range errs { + log.Err(err).Send() + } + }() + + go func(r io.Reader) { + rdr, err := h264reader.NewReader(r) + if err != nil { + log.Error().Err(err).Msg("failed to create ivfreader") + return + } + + v.mu.Lock() + close(v.ivfCodecReady) + v.mu.Unlock() + + log.Info().Msgf("starting to stream H264") + ticker := time.NewTicker(time.Millisecond * 33) + defer ticker.Stop() + + for ; true; <-ticker.C { + select { + case <-v.ctx.Done(): + // Exit cleanly + return + default: + // do nothing + } + frame, ivfErr := rdr.NextNAL() + if errors.Is(ivfErr, io.EOF) { + log.Debug().Msg("all video frames parsed and sent") + return + } + if ivfErr != nil { + log.Error().Err(ivfErr).Msg("failed to parse frame") + } + if frame == nil { + log.Debug().Msg("all video frames parsed and sent") + return + } + + v.mu.Lock() + for _, lis := range v.ivfListeners { + lis <- frame.Data + } + v.mu.Unlock() + + } + }(h264snoop) + log.Info().Msg("video streams started") + + <-v.ctx.Done() +} + +// Join will connect to a running stream; note, it will block +// until the ivf codec is decided. Make sure to call Run first. +func (v *Video) Join() (<-chan []byte, string, func()) { + <-v.ivfCodecReady + v.mu.Lock() + defer v.mu.Unlock() + + myID := uuid.New().String() + ch := make(chan []byte) + v.ivfListeners[myID] = ch + + return ch, webrtc.MimeTypeH264, func() { + v.mu.Lock() + defer v.mu.Unlock() + + delete(v.ivfListeners, myID) + } +} + +func logToStdErr(name string, w *io.Writer) { + pipeReader, pipeWriter := io.Pipe() + *w = pipeWriter + + lineReader := bufio.NewScanner(pipeReader) + + for lineReader.Scan() { + log.Info().Str("video", name).Str("stderr", lineReader.Text()).Send() + } +} + +// Done stops the processing. +func (v *Video) Done() { + v.cancelFunc() +} + +type Mod struct{} + +func (m *Mod) Get() (*Video, error) { + cfg, err := config.Default.Get() + if err != nil { + return nil, err + } + return New(cfg) +} diff --git a/proto/signaler_service.proto b/proto/signaler_service.proto index 8f54baf..0ee6952 100644 --- a/proto/signaler_service.proto +++ b/proto/signaler_service.proto @@ -64,7 +64,7 @@ message CreateSessionRequest { } message PopSessionRequest { - Session session = 1; + optional Session session = 1; } message UpdateSessionRequest { diff --git a/sample.yaml b/sample.yaml index b9c7adb..64b5a39 100644 --- a/sample.yaml +++ b/sample.yaml @@ -3,22 +3,10 @@ name: Movie h264: binary: "/usr/bin/cat" arguments: - - "/home/charles/Downloads/simpsons_movie_1080p_hddvd_trailer/The Simpsons Movie - 1080p Trailer.mp4" -ivf: - binary: "/usr/bin/ffmpeg" - arguments: - - "-i" - - "-" - - "-g" - - "30" - - "-b:v" - - "2M" - - "-f" - - "ivf" - - "-" + - "/home/charles/Downloads/big_buck_bunny_1080p.h264" sensor: binary: "/usr/bin/yes" arguments: - "44.33889713096721 1015.1977693083448 23.60542243081727" -sensor_rate_ms: 10000 \ No newline at end of file +sensor_rate_ms: 10000 diff --git a/ui/lib/main.dart b/ui/lib/main.dart index 231a6b4..ef3e5b6 100644 --- a/ui/lib/main.dart +++ b/ui/lib/main.dart @@ -11,14 +11,20 @@ void main() async { logger.i("Establishing connection..."); final channel = ClientChannel( 'home.chathaway.codes', - //port: 80, + //port: 8080, //options: const ChannelOptions(credentials: ChannelCredentials.insecure()), channelShutdownHandler: () { logger.e("Channel shutdown unexpectedly"); }, ); + logger.i("Have connection, making stub"); final stub = SignalerServiceClient(channel); - runApp(MyApp(stub, SessionService(stub))); + final sessionService = SessionService(stub); + logger.i("Have stub, starting app"); + runApp(MyApp( + stub, + sessionService, + )); } class MyApp extends StatelessWidget { @@ -32,21 +38,6 @@ class MyApp extends StatelessWidget { return MaterialApp( title: 'Flutter Demo', theme: ThemeData( - // This is the theme of your application. - // - // TRY THIS: Try running your application with "flutter run". You'll see - // the application has a blue toolbar. Then, without quitting the app, - // try changing the seedColor in the colorScheme below to Colors.green - // and then invoke "hot reload" (save your changes or press the "hot - // reload" button in a Flutter-supported IDE, or press "r" if you used - // the command line to start the app). - // - // Notice that the counter didn't reset back to zero; the application - // state is not lost during the reload. To reset the state, use hot - // restart instead. - // - // This works for code too, not just values: Most code changes can be - // tested with just a hot reload. colorScheme: ColorScheme.fromSeed(seedColor: Colors.deepPurple), useMaterial3: true, ), diff --git a/ui/pubspec.lock b/ui/pubspec.lock index 79650b9..f97f6d0 100644 --- a/ui/pubspec.lock +++ b/ui/pubspec.lock @@ -5,10 +5,10 @@ packages: dependency: transitive description: name: archive - sha256: "1227dc3efc4ea571eebb2dfb814506ed2cfb1d4b1b89fb918abdddde617ead3c" + sha256: "22600aa1e926be775fa5fe7e6894e7fb3df9efda8891c73f70fb3262399a432d" url: "https://pub.dev" source: hosted - version: "3.4.0" + version: "3.4.10" args: dependency: transitive description: @@ -53,10 +53,10 @@ packages: dependency: transitive description: name: collection - sha256: f092b211a4319e98e5ff58223576de6c2803db36221657b46c82574721240687 + sha256: ee67cb0715911d28db6bf4af1026078bd6f0128b07a5f66fb2ed94ec6783c09a url: "https://pub.dev" source: hosted - version: "1.17.2" + version: "1.18.0" convert: dependency: transitive description: @@ -135,10 +135,10 @@ packages: dependency: "direct main" description: name: flutter_webrtc - sha256: f952d207725ea66f36f1e59898a8026645d1e0e02f1455710289f75f09e458e5 + sha256: "577216727181cb13776a65d3e7cb33e783e740c5496335011aed4a038b28c3fe" url: "https://pub.dev" source: hosted - version: "0.9.42+hotfix.1" + version: "0.9.47" googleapis_auth: dependency: transitive description: @@ -159,10 +159,10 @@ packages: dependency: transitive description: name: http - sha256: "759d1a329847dd0f39226c688d3e06a6b8679668e350e2891a6474f8b4bb8525" + sha256: d4872660c46d929f6b8a9ef4e7a7eff7e49bbf0c4ec3f385ee32df5119175139 url: "https://pub.dev" source: hosted - version: "1.1.0" + version: "1.1.2" http2: dependency: transitive description: @@ -199,10 +199,10 @@ packages: dependency: "direct main" description: name: logger - sha256: ba3bc83117b2b49bdd723c0ea7848e8285a0fbc597ba09203b20d329d020c24a + sha256: "6bbb9d6f7056729537a4309bda2e74e18e5d9f14302489cc1e93f33b3fe32cac" url: "https://pub.dev" source: hosted - version: "2.0.2" + version: "2.0.2+1" matcher: dependency: transitive description: @@ -223,10 +223,10 @@ packages: dependency: transitive description: name: meta - sha256: "3c74dbf8763d36539f114c799d8a2d87343b5067e9d796ca22b5eb8437090ee3" + sha256: a6e590c838b18133bb482a2745ad77c5bb7715fb0451209e1a7567d416678b8e url: "https://pub.dev" source: hosted - version: "1.9.1" + version: "1.10.0" path: dependency: transitive description: @@ -247,10 +247,10 @@ packages: dependency: transitive description: name: path_provider_android - sha256: "6b8b19bd80da4f11ce91b2d1fb931f3006911477cec227cce23d3253d80df3f1" + sha256: "477184d672607c0a3bf68fbbf601805f92ef79c82b64b4d6eb318cbca4c48668" url: "https://pub.dev" source: hosted - version: "2.2.0" + version: "2.2.2" path_provider_foundation: dependency: transitive description: @@ -287,26 +287,26 @@ packages: dependency: transitive description: name: platform - sha256: ae68c7bfcd7383af3629daafb32fb4e8681c7154428da4febcff06200585f102 + sha256: "12220bb4b65720483f8fa9450b4332347737cf8213dd2840d8b2c823e47243ec" url: "https://pub.dev" source: hosted - version: "3.1.2" + version: "3.1.4" platform_detect: dependency: transitive description: name: platform_detect - sha256: "14afcb6ffcd93745e39a288db53d1d6522ea25d71f7993c13a367a86c437b54d" + sha256: "08f4ee79c0e1c4858d37e06b22352a3ebdef5466b613749a3adb03e703d4f5b0" url: "https://pub.dev" source: hosted - version: "2.0.7" + version: "2.0.11" plugin_platform_interface: dependency: transitive description: name: plugin_platform_interface - sha256: da3fdfeccc4d4ff2da8f8c556704c08f912542c5fb3cf2233ed75372384a034d + sha256: "4820fbfdb9478b1ebae27888254d445073732dae3d6ea81f0b7e06d5dedc3f02" url: "https://pub.dev" source: hosted - version: "2.1.6" + version: "2.1.8" pointycastle: dependency: transitive description: @@ -348,18 +348,18 @@ packages: dependency: transitive description: name: stack_trace - sha256: c3c7d8edb15bee7f0f74debd4b9c5f3c2ea86766fe4178eb2a18eb30a0bdaed5 + sha256: "73713990125a6d93122541237550ee3352a2d84baad52d375a4cad2eb9b7ce0b" url: "https://pub.dev" source: hosted - version: "1.11.0" + version: "1.11.1" stream_channel: dependency: transitive description: name: stream_channel - sha256: "83615bee9045c1d322bbbd1ba209b7a749c2cbcdcb3fdd1df8eb488b3279c1c8" + sha256: ba2aa5d8cc609d96bbb2899c28934f9e1af5cddbd60a827822ea467161eb54e7 url: "https://pub.dev" source: hosted - version: "2.1.1" + version: "2.1.2" string_scanner: dependency: transitive description: @@ -380,10 +380,10 @@ packages: dependency: transitive description: name: test_api - sha256: "75760ffd7786fffdfb9597c35c5b27eaeec82be8edfb6d71d32651128ed7aab8" + sha256: "5c2f730018264d276c20e4f1503fd1308dfbbae39ec8ee63c5236311ac06954b" url: "https://pub.dev" source: hosted - version: "0.6.0" + version: "0.6.1" typed_data: dependency: transitive description: @@ -404,10 +404,10 @@ packages: dependency: transitive description: name: web - sha256: dc8ccd225a2005c1be616fe02951e2e342092edf968cf0844220383757ef8f10 + sha256: afe077240a270dcfd2aafe77602b4113645af95d0ad31128cc02bce5ac5d5152 url: "https://pub.dev" source: hosted - version: "0.1.4-beta" + version: "0.3.0" webrtc_interface: dependency: transitive description: @@ -420,18 +420,18 @@ packages: dependency: transitive description: name: win32 - sha256: "9e82a402b7f3d518fb9c02d0e9ae45952df31b9bf34d77baf19da2de03fc2aaa" + sha256: "464f5674532865248444b4c3daca12bd9bf2d7c47f759ce2617986e7229494a8" url: "https://pub.dev" source: hosted - version: "5.0.7" + version: "5.2.0" xdg_directories: dependency: transitive description: name: xdg_directories - sha256: "589ada45ba9e39405c198fe34eb0f607cddb2108527e658136120892beac46d2" + sha256: faea9dee56b520b55a566385b84f2e8de55e7496104adada9962e0bd11bcff1d url: "https://pub.dev" source: hosted - version: "1.0.3" + version: "1.0.4" sdks: - dart: ">=3.1.2 <4.0.0" - flutter: ">=3.7.0" + dart: ">=3.2.0 <4.0.0" + flutter: ">=3.10.0"