diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1213661 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +invidious/ \ No newline at end of file diff --git a/1-2469.patch b/1-2469.patch new file mode 100644 index 0000000..2ce4361 --- /dev/null +++ b/1-2469.patch @@ -0,0 +1,358 @@ +From 6a19f66c5380488896c341d88a52b99601008d8c Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?=C3=89milien=20Devos?= +Date: Tue, 19 Oct 2021 07:12:15 +0000 +Subject: [PATCH] limit feeds and delete materialized views + +--- + config/migrate-scripts/migrate-db-8bc91ce.sh | 6 ++ + config/sql/channel_videos.sql | 8 +-- + kubernetes/values.yaml | 1 - + src/invidious.cr | 12 ---- + src/invidious/config.cr | 2 - + src/invidious/jobs/refresh_feeds_job.cr | 75 -------------------- + src/invidious/routes/account.cr | 2 - + src/invidious/routes/login.cr | 3 - + src/invidious/routes/search.cr | 2 + + src/invidious/search/processors.cr | 18 ++--- + src/invidious/users.cr | 51 ++++++------- + 11 files changed, 43 insertions(+), 137 deletions(-) + create mode 100644 config/migrate-scripts/migrate-db-8bc91ce.sh + delete mode 100644 src/invidious/jobs/refresh_feeds_job.cr + +diff --git a/config/migrate-scripts/migrate-db-8bc91ce.sh b/config/migrate-scripts/migrate-db-8bc91ce.sh +new file mode 100644 +index 000000000..04388175e +--- /dev/null ++++ b/config/migrate-scripts/migrate-db-8bc91ce.sh +@@ -0,0 +1,6 @@ ++CREATE INDEX channel_videos_ucid_published_idx ++ ON public.channel_videos ++ USING btree ++ (ucid COLLATE pg_catalog."default", published); ++ ++DROP INDEX channel_videos_ucid_idx; +\ No newline at end of file +diff --git a/config/sql/channel_videos.sql b/config/sql/channel_videos.sql +index cd4e0ffdb..f2ac4876c 100644 +--- a/config/sql/channel_videos.sql ++++ b/config/sql/channel_videos.sql +@@ -19,12 +19,12 @@ CREATE TABLE IF NOT EXISTS public.channel_videos + + GRANT ALL ON TABLE public.channel_videos TO current_user; + +--- Index: public.channel_videos_ucid_idx ++-- Index: public.channel_videos_ucid_published_idx + +--- DROP INDEX public.channel_videos_ucid_idx; ++-- DROP INDEX public.channel_videos_ucid_published_idx; + +-CREATE INDEX IF NOT EXISTS channel_videos_ucid_idx ++CREATE INDEX IF NOT EXISTS channel_videos_ucid_published_idx + ON public.channel_videos + USING btree +- (ucid COLLATE pg_catalog."default"); ++ (ucid COLLATE pg_catalog."default", published); + +diff --git a/kubernetes/values.yaml b/kubernetes/values.yaml +index 2dc4db2c4..5506c772d 100644 +--- a/kubernetes/values.yaml ++++ b/kubernetes/values.yaml +@@ -49,7 +49,6 @@ postgresql: + # Adapted from ../config/config.yml + config: + channel_threads: 1 +- feed_threads: 1 + db: + user: kemal + password: kemal +diff --git a/src/invidious.cr b/src/invidious.cr +index 9f3d5d10f..6c6bd32ce 100644 +--- a/src/invidious.cr ++++ b/src/invidious.cr +@@ -86,14 +86,6 @@ Kemal.config.extra_options do |parser| + exit + end + end +- parser.on("-f THREADS", "--feed-threads=THREADS", "Number of threads for refreshing feeds (default: #{CONFIG.feed_threads})") do |number| +- begin +- CONFIG.feed_threads = number.to_i +- rescue ex +- puts "THREADS must be integer" +- exit +- end +- end + parser.on("-o OUTPUT", "--output=OUTPUT", "Redirect output (default: #{CONFIG.output})") do |output| + CONFIG.output = output + end +@@ -141,10 +133,6 @@ if CONFIG.channel_threads > 0 + Invidious::Jobs.register Invidious::Jobs::RefreshChannelsJob.new(PG_DB) + end + +-if CONFIG.feed_threads > 0 +- Invidious::Jobs.register Invidious::Jobs::RefreshFeedsJob.new(PG_DB) +-end +- + DECRYPT_FUNCTION = DecryptFunction.new(CONFIG.decrypt_polling) + if CONFIG.decrypt_polling + Invidious::Jobs.register Invidious::Jobs::UpdateDecryptFunctionJob.new +diff --git a/src/invidious/config.cr b/src/invidious/config.cr +index 93c4c0f7a..6fe72dbda 100644 +--- a/src/invidious/config.cr ++++ b/src/invidious/config.cr +@@ -62,8 +62,6 @@ class Config + # Time interval between two executions of the job that crawls channel videos (subscriptions update). + @[YAML::Field(converter: Preferences::TimeSpanConverter)] + property channel_refresh_interval : Time::Span = 30.minutes +- # Number of threads to use for updating feeds +- property feed_threads : Int32 = 1 + # Log file path or STDOUT + property output : String = "STDOUT" + # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr +diff --git a/src/invidious/jobs/refresh_feeds_job.cr b/src/invidious/jobs/refresh_feeds_job.cr +deleted file mode 100644 +index 4b52c9596..000000000 +--- a/src/invidious/jobs/refresh_feeds_job.cr ++++ /dev/null +@@ -1,75 +0,0 @@ +-class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob +- private getter db : DB::Database +- +- def initialize(@db) +- end +- +- def begin +- max_fibers = CONFIG.feed_threads +- active_fibers = 0 +- active_channel = Channel(Bool).new +- +- loop do +- db.query("SELECT email FROM users WHERE feed_needs_update = true OR feed_needs_update IS NULL") do |rs| +- rs.each do +- email = rs.read(String) +- view_name = "subscriptions_#{sha256(email)}" +- +- if active_fibers >= max_fibers +- if active_channel.receive +- active_fibers -= 1 +- end +- end +- +- active_fibers += 1 +- spawn do +- begin +- # Drop outdated views +- column_array = Invidious::Database.get_column_array(db, view_name) +- ChannelVideo.type_array.each_with_index do |name, i| +- if name != column_array[i]? +- LOGGER.info("RefreshFeedsJob: DROP MATERIALIZED VIEW #{view_name}") +- db.exec("DROP MATERIALIZED VIEW #{view_name}") +- raise "view does not exist" +- end +- end +- +- if !db.query_one("SELECT pg_get_viewdef('#{view_name}')", as: String).includes? "WHERE ((cv.ucid = ANY (u.subscriptions))" +- LOGGER.info("RefreshFeedsJob: Materialized view #{view_name} is out-of-date, recreating...") +- db.exec("DROP MATERIALIZED VIEW #{view_name}") +- end +- +- db.exec("REFRESH MATERIALIZED VIEW #{view_name}") +- db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email) +- rescue ex +- # Rename old views +- begin +- legacy_view_name = "subscriptions_#{sha256(email)[0..7]}" +- +- db.exec("SELECT * FROM #{legacy_view_name} LIMIT 0") +- LOGGER.info("RefreshFeedsJob: RENAME MATERIALIZED VIEW #{legacy_view_name}") +- db.exec("ALTER MATERIALIZED VIEW #{legacy_view_name} RENAME TO #{view_name}") +- rescue ex +- begin +- # While iterating through, we may have an email stored from a deleted account +- if db.query_one?("SELECT true FROM users WHERE email = $1", email, as: Bool) +- LOGGER.info("RefreshFeedsJob: CREATE #{view_name}") +- db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(email)}") +- db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email) +- end +- rescue ex +- LOGGER.error("RefreshFeedJobs: REFRESH #{email} : #{ex.message}") +- end +- end +- end +- +- active_channel.send(true) +- end +- end +- end +- +- sleep 5.seconds +- Fiber.yield +- end +- end +-end +diff --git a/src/invidious/routes/account.cr b/src/invidious/routes/account.cr +index 9bb731360..e3220cdb8 100644 +--- a/src/invidious/routes/account.cr ++++ b/src/invidious/routes/account.cr +@@ -128,10 +128,8 @@ module Invidious::Routes::Account + return error_template(400, ex) + end + +- view_name = "subscriptions_#{sha256(user.email)}" + Invidious::Database::Users.delete(user) + Invidious::Database::SessionIDs.delete(email: user.email) +- PG_DB.exec("DROP MATERIALIZED VIEW #{view_name}") + + env.request.cookies.each do |cookie| + cookie.expires = Time.utc(1990, 1, 1) +diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr +index 99fc13a2b..ca223b425 100644 +--- a/src/invidious/routes/login.cr ++++ b/src/invidious/routes/login.cr +@@ -430,9 +430,6 @@ module Invidious::Routes::Login + Invidious::Database::Users.insert(user) + Invidious::Database::SessionIDs.insert(sid, email) + +- view_name = "subscriptions_#{sha256(user.email)}" +- PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") +- + env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.domain, sid) + + if env.request.cookies["PREFS"]? +diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr +index e60d00815..1d4911bde 100644 +--- a/src/invidious/routes/search.cr ++++ b/src/invidious/routes/search.cr +@@ -51,6 +51,8 @@ module Invidious::Routes::Search + else + user = env.get? "user" + ++ user = user ? user.as(User) : nil ++ + begin + videos = query.process + rescue ex : ChannelSearchException +diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr +index d1409c06c..1ff6f95a9 100644 +--- a/src/invidious/search/processors.cr ++++ b/src/invidious/search/processors.cr +@@ -45,18 +45,18 @@ module Invidious::Search + + # Search inside of user subscriptions + def subscriptions(query : Query, user : Invidious::User) : Array(ChannelVideo) +- view_name = "subscriptions_#{sha256(user.email)}" +- + return PG_DB.query_all(" + SELECT id,title,published,updated,ucid,author,length_seconds + FROM ( +- SELECT *, +- to_tsvector(#{view_name}.title) || +- to_tsvector(#{view_name}.author) +- as document +- FROM #{view_name} +- ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", +- query.text, (query.page - 1) * 20, ++ SELECT cv.*, ++ to_tsvector(cv.title) || ++ to_tsvector(cv.author) AS document ++ FROM channel_videos cv ++ JOIN users ON cv.ucid = any(users.subscriptions) ++ WHERE users.email = $1 AND published > now() - interval '1 month' ++ ORDER BY published ++ ) v_search WHERE v_search.document @@ plainto_tsquery($2) LIMIT 20 OFFSET $3;", ++ user.email, query.text, (query.page - 1) * 20, + as: ChannelVideo + ) + end +diff --git a/src/invidious/users.cr b/src/invidious/users.cr +index b763596bc..6f82ead33 100644 +--- a/src/invidious/users.cr ++++ b/src/invidious/users.cr +@@ -12,24 +12,12 @@ def get_user(sid, headers, refresh = true) + + Invidious::Database::Users.insert(user, update_on_conflict: true) + Invidious::Database::SessionIDs.insert(sid, user.email, handle_conflicts: true) +- +- begin +- view_name = "subscriptions_#{sha256(user.email)}" +- PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") +- rescue ex +- end + end + else + user, sid = fetch_user(sid, headers) + + Invidious::Database::Users.insert(user, update_on_conflict: true) + Invidious::Database::SessionIDs.insert(sid, user.email, handle_conflicts: true) +- +- begin +- view_name = "subscriptions_#{sha256(user.email)}" +- PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") +- rescue ex +- end + end + + return user, sid +@@ -128,7 +116,6 @@ def get_subscription_feed(user, max_results = 40, page = 1) + offset = (page - 1) * limit + + notifications = Invidious::Database::Users.select_notifications(user) +- view_name = "subscriptions_#{sha256(user.email)}" + + if user.preferences.notifications_only && !notifications.empty? + # Only show notifications +@@ -154,33 +141,39 @@ def get_subscription_feed(user, max_results = 40, page = 1) + # Show latest video from a channel that a user hasn't watched + # "unseen_only" isn't really correct here, more accurate would be "unwatched_only" + +- if user.watched.empty? +- values = "'{}'" +- else +- values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}" +- end +- videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY ucid, published DESC", as: ChannelVideo) ++ # "SELECT cv.* FROM channel_videos cv JOIN users ON cv.ucid = any(users.subscriptions) WHERE users.email = $1 AND published > now() - interval '1 month' ORDER BY published DESC" ++ # "SELECT DISTINCT ON (cv.ucid) cv.* FROM channel_videos cv JOIN users ON cv.ucid = any(users.subscriptions) WHERE users.email = ? AND NOT cv.id = any(users.watched) AND published > now() - interval '1 month' ORDER BY ucid, published DESC" ++ videos = PG_DB.query_all("SELECT DISTINCT ON (cv.ucid) cv.* " \ ++ "FROM channel_videos cv " \ ++ "JOIN users ON cv.ucid = any(users.subscriptions) " \ ++ "WHERE users.email = $1 AND NOT cv.id = any(users.watched) AND published > now() - interval '1 month' " \ ++ "ORDER BY ucid, published DESC", user.email, as: ChannelVideo) + else + # Show latest video from each channel + +- videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} ORDER BY ucid, published DESC", as: ChannelVideo) ++ videos = PG_DB.query_all("SELECT DISTINCT ON (cv.ucid) cv.* " \ ++ "FROM channel_videos cv " \ ++ "JOIN users ON cv.ucid = any(users.subscriptions) " \ ++ "WHERE users.email = $1 AND published > now() - interval '1 month' " \ ++ "ORDER BY ucid, published DESC", user.email, as: ChannelVideo) + end + + videos.sort_by!(&.published).reverse! + else + if user.preferences.unseen_only + # Only show unwatched +- +- if user.watched.empty? +- values = "'{}'" +- else +- values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}" +- end +- videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo) ++ videos = PG_DB.query_all("SELECT cv.* " \ ++ "FROM channel_videos cv " \ ++ "JOIN users ON cv.ucid = any(users.subscriptions) " \ ++ "WHERE users.email = $1 AND NOT cv.id = any(users.watched) AND published > now() - interval '1 month' " \ ++ "ORDER BY published DESC LIMIT $2 OFFSET $3", user.email, limit, offset, as: ChannelVideo) + else + # Sort subscriptions as normal +- +- videos = PG_DB.query_all("SELECT * FROM #{view_name} ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo) ++ videos = PG_DB.query_all("SELECT cv.* " \ ++ "FROM channel_videos cv " \ ++ "JOIN users ON cv.ucid = any(users.subscriptions) " \ ++ "WHERE users.email = $1 AND published > now() - interval '1 month' " \ ++ "ORDER BY published DESC LIMIT $2 OFFSET $3", user.email, limit, offset, as: ChannelVideo) + end + end + diff --git a/2-2111.patch.disable b/2-2111.patch.disable new file mode 100644 index 0000000..9ad24ac --- /dev/null +++ b/2-2111.patch.disable @@ -0,0 +1,45 @@ +From 78773d732672d8985795fb040a39dd7e946c7b7c Mon Sep 17 00:00:00 2001 +From: Emilien Devos +Date: Sat, 22 May 2021 17:42:23 +0200 +Subject: [PATCH] add the ability to listen on unix sockets + +--- + src/invidious.cr | 15 ++++++++++++--- + src/invidious/helpers/helpers.cr | 1 + + 2 files changed, 13 insertions(+), 3 deletions(-) + +diff --git a/src/invidious.cr b/src/invidious.cr +index ae20e13e5..65b1091bb 100644 +--- a/src/invidious.cr ++++ b/src/invidious.cr +@@ -3917,6 +3917,15 @@ add_context_storage_type(Preferences) + add_context_storage_type(User) + + Kemal.config.logger = LOGGER +-Kemal.config.host_binding = Kemal.config.host_binding != "0.0.0.0" ? Kemal.config.host_binding : CONFIG.host_binding +-Kemal.config.port = Kemal.config.port != 3000 ? Kemal.config.port : CONFIG.port +-Kemal.run ++ ++Kemal.run do |config| ++ if CONFIG.bind_unix ++ if File.exists?(CONFIG.bind_unix.not_nil!) ++ File.delete(CONFIG.bind_unix.not_nil!) ++ end ++ config.server.not_nil!.bind_unix CONFIG.bind_unix.not_nil! ++ else ++ config.host_binding = config.host_binding != "0.0.0.0" ? config.host_binding : CONFIG.host_binding ++ config.port = config.port != 3000 ? config.port : CONFIG.port ++ end ++end +diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr +index e1d877b78..6a5789a0e 100644 +--- a/src/invidious/helpers/helpers.cr ++++ b/src/invidious/helpers/helpers.cr +@@ -98,6 +98,7 @@ class Config + property force_resolve : Socket::Family = Socket::Family::UNSPEC # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729) + property port : Int32 = 3000 # Port to listen for connections (overrided by command line argument) + property host_binding : String = "0.0.0.0" # Host to bind (overrided by command line argument) ++ property bind_unix : String? = nil # Make Invidious listening on UNIX sockets - Example: /tmp/invidious.sock + property pool_size : Int32 = 100 # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) + property use_quic : Bool = true # Use quic transport for youtube api + diff --git a/APKBUILD-boringssl b/APKBUILD-boringssl new file mode 100644 index 0000000..61caa4f --- /dev/null +++ b/APKBUILD-boringssl @@ -0,0 +1,46 @@ +# Based on https://aur.archlinux.org/packages/boringssl-git/ +# Maintainer: Omar Roth +pkgname=boringssl +pkgver=1.1.0 +pkgrel=0 +pkgdesc="BoringSSL is a fork of OpenSSL that is designed to meet Google's needs" +url="https://boringssl.googlesource.com/boringssl" +arch="all" +license="MIT" +replaces="openssl libressl" +depends="!openssl-libs-static" +makedepends_host="linux-headers" +makedepends="cmake git go perl" +subpackages="$pkgname-static $pkgname-dev $pkgname-doc" +source="251b516.tar.gz::https://github.com/google/boringssl/tarball/251b516" +builddir="$srcdir/google-boringssl-251b516" + +prepare() { + : +} + +build() { + cmake -DCMAKE_BUILD_TYPE=Release . + make ssl crypto +} + +check() { + make all_tests +} + +package() { + for i in *.md ; do + install -Dm644 $i "$pkgdir/usr/share/doc/$pkgname/$i" + done + install -d "$pkgdir/usr/lib" + install -d "$pkgdir/usr/include" + cp -R include/openssl "$pkgdir/usr/include" + + install -Dm755 crypto/libcrypto.a "$pkgdir/usr/lib/libcrypto.a" + install -Dm755 ssl/libssl.a "$pkgdir/usr/lib/libssl.a" +# install -Dm755 decrepit/libdecrepit.a "$pkgdir/usr/lib/libdecrepit.a" +# install -Dm755 libboringssl_gtest.a "$pkgdir/usr/lib/libboringssl_gtest.a" +} +sha512sums=" +b1d42ed188cf0cce89d40061fa05de85b387ee4244f1236ea488a431536a2c6b657b4f03daed0ac9328c7f5c4c9330499283b8a67f1444dcf9ba5e97e1199c4e 251b516.tar.gz +" diff --git a/APKBUILD-lsquic b/APKBUILD-lsquic new file mode 100644 index 0000000..51630a0 --- /dev/null +++ b/APKBUILD-lsquic @@ -0,0 +1,43 @@ +# Maintainer: Omar Roth +pkgname=lsquic +pkgver=2.18.1 +pkgrel=0 +pkgdesc="LiteSpeed QUIC and HTTP/3 Library" +url="https://github.com/litespeedtech/lsquic" +arch="all" +license="MIT" +depends="boringssl-dev boringssl-static zlib-static libevent-static" +makedepends="cmake git go perl bsd-compat-headers linux-headers" +subpackages="$pkgname-static" +source="v$pkgver.tar.gz::https://github.com/litespeedtech/lsquic/tarball/v2.18.1 +ls-qpack-$pkgver.tar.gz::https://github.com/litespeedtech/ls-qpack/tarball/a8ae6ef +ls-hpack-$pkgver.tar.gz::https://github.com/litespeedtech/ls-hpack/tarball/bd5d589" +builddir="$srcdir/litespeedtech-$pkgname-692a910" + +prepare() { + cp -r -T "$srcdir/litespeedtech-ls-qpack-a8ae6ef" "$builddir/src/liblsquic/ls-qpack" + cp -r -T "$srcdir/litespeedtech-ls-hpack-bd5d589" "$builddir/src/lshpack" +} + +build() { + cmake \ + -DCMAKE_BUILD_TYPE=None \ + -DBORINGSSL_INCLUDE=/usr/include/openssl \ + -DBORINGSSL_LIB_crypto=/usr/lib \ + -DBORINGSSL_LIB_ssl=/usr/lib . + make lsquic +} + +check() { + make tests +} + +package() { + install -d "$pkgdir/usr/lib" + install -Dm755 src/liblsquic/liblsquic.a "$pkgdir/usr/lib/liblsquic.a" +} +sha512sums=" +d015a72f1e88750ecb364768a40f532678f11ded09c6447a2e698b20f43fa499ef143a53f4c92a5938dfece0e39e687dc9df4aea97c618faee0c63da771561c3 v2.18.1.tar.gz +c5629085a3881815fb0b72a321eeba8de093eff9417b8ac7bde1ee1264971be0dca6d61d74799b02ae03a4c629b2a9cf21387deeb814935339a8a2503ea33fee ls-qpack-2.18.1.tar.gz +1b9f7ce4c82dadfca8154229a415b0335a61761eba698f814d4b94195c708003deb5cb89318a1ab78ac8fa88b141bc9df283fb1c6e40b3ba399660feaae353a0 ls-hpack-2.18.1.tar.gz +" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3545f54 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,39 @@ +FROM crystallang/crystal:1.4.1-alpine AS builder +RUN apk add --no-cache sqlite-static yaml-static + +WORKDIR /invidious +COPY ./invidious/shard.yml ./shard.yml +COPY ./invidious/shard.lock ./shard.lock +RUN shards update --production && shards install --production + +COPY --from=quay.io/invidious/lsquic-compiled /root/liblsquic.a ./lib/lsquic/src/lsquic/ext/liblsquic.a + +COPY ./invidious/src/ ./src/ +# TODO: .git folder is required for building – this is destructive. +# See definition of CURRENT_BRANCH, CURRENT_COMMIT and CURRENT_VERSION. +COPY ./invidious/.git/ ./.git/ +# Required for fetching player dependencies +COPY ./invidious/scripts/ ./scripts/ +COPY ./invidious/assets/ ./assets/ +COPY ./invidious/videojs-dependencies.yml ./videojs-dependencies.yml +RUN crystal build --release ./src/invidious.cr \ + --static --warnings all \ + --link-flags "-lxml2 -llzma" + +FROM alpine:latest +RUN apk add --no-cache librsvg ttf-opensans +WORKDIR /invidious +RUN addgroup -g 1000 -S invidious && \ + adduser -u 1000 -S invidious -G invidious +COPY --chown=invidious ./invidious/config/config.* ./config/ +RUN mv -n config/config.example.yml config/config.yml +RUN sed -i 's/host: \(127.0.0.1\|localhost\)/host: postgres/' config/config.yml +COPY ./invidious/config/sql/ ./config/sql/ +COPY ./invidious/locales/ ./locales/ +COPY --from=builder /invidious/assets ./assets/ +COPY --from=builder /invidious/invidious . +RUN chmod o+rX -R ./assets ./config ./locales + +EXPOSE 3000 +USER invidious +CMD [ "/invidious/invidious" ] \ No newline at end of file diff --git a/Dockerfile.new b/Dockerfile.new new file mode 100644 index 0000000..cb1a914 --- /dev/null +++ b/Dockerfile.new @@ -0,0 +1,49 @@ +FROM alpine:edge AS builder +RUN apk add --no-cache 'crystal=1.4.1-r0' shards sqlite-static yaml-static yaml-dev libxml2-dev zlib-static openssl-libs-static openssl-dev musl-dev yq + +ARG add_build_args + +WORKDIR /invidious +COPY ./invidious/shard.yml ./shard.yml +COPY ./invidious/shard.lock ./shard.lock +# Sentry is just for reporting Invidious crashes, no personal data is collected. +#RUN yq e -i '.dependencies.raven.github = "Sija/raven.cr"' shard.yml +#RUN yq e -i '.targets.sentry_crash_handler.main = "lib/raven/src/crash_handler.cr"' shard.yml +RUN shards install --production + +COPY --from=quay.io/invidious/lsquic-compiled /root/liblsquic.a ./lib/lsquic/src/lsquic/ext/liblsquic.a + +COPY ./invidious/src/ ./src/ +# TODO: .git folder is required for building – this is destructive. +# See definition of CURRENT_BRANCH, CURRENT_COMMIT and CURRENT_VERSION. +COPY ./invidious/.git/ ./.git/ +# Required for fetching player dependencies +COPY ./invidious/scripts/ ./scripts/ +COPY ./invidious/assets/ ./assets/ +COPY ./invidious/videojs-dependencies.yml ./videojs-dependencies.yml +RUN crystal build ./src/invidious.cr ${add_build_args} \ + --release \ + --static --warnings all \ + --link-flags "-lxml2 -llzma"; + +#RUN shards build --release --static sentry_crash_handler + +FROM alpine:edge +RUN apk add --no-cache librsvg ttf-opensans +WORKDIR /invidious +RUN addgroup -g 1000 -S invidious && \ + adduser -u 1000 -S invidious -G invidious +COPY --chown=invidious ./invidious/config/config.* ./config/ +RUN mv -n config/config.example.yml config/config.yml +RUN sed -i 's/host: \(127.0.0.1\|localhost\)/host: postgres/' config/config.yml +COPY ./invidious/config/sql/ ./config/sql/ +COPY ./invidious/locales/ ./locales/ +COPY --from=builder /invidious/assets ./assets/ +COPY --from=builder /invidious/invidious . +#COPY --from=builder /invidious/bin/sentry_crash_handler . +RUN chmod o+rX -R ./assets ./config ./locales + +EXPOSE 3000 +USER invidious +#CMD [ "/invidious/sentry_crash_handler", "/invidious/invidious" ] +CMD [ "/invidious/invidious" ] diff --git a/docker-image.yml b/docker-image.yml new file mode 100644 index 0000000..93e26d0 --- /dev/null +++ b/docker-image.yml @@ -0,0 +1,99 @@ +name: Docker Image CI + +on: + push: + branches: + - master + + pull_request: + branches: + - master + + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 0 * * *' + +jobs: + + build: + + runs-on: ubuntu-latest + + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + with: + platforms: arm64 + - name: Check Out Repo + uses: actions/checkout@v2 + + - name: Check Out Repo Invidious + uses: actions/checkout@v2 + with: + repository: "iv-org/invidious" + ref: 'master' + path: 'invidious' + + - name: patch invidious main repo + run: | + cd invidious + git config --global user.email "you@example.com" + git config --global user.name "Your Name" + git am ../patches/*.patch + sed -i 's/https:\/\/invidious.io\/donate\//\/donate/g' src/invidious/views/template.ecr + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + + - name: Set up build commit id + run: echo "commitid=$(cd invidious && git log --format="%H" -n 1)-$(git log --format="%H" -n 1)" >> $GITHUB_ENV + + - name: Build and push alpine docker image + id: docker_build_new + uses: docker/build-push-action@v2 + with: + context: ./ + file: ./Dockerfile.new + build-args: | + release=1 + platforms: linux/amd64,linux/arm64/v8 + push: true + tags: unixfox/invidious-custom:new-latest, unixfox/invidious-custom:new-build-${{ env.commitid }} + + - name: patch invidious API repo + run: | + cd invidious + git config --global user.email "you@example.com" + git config --global user.name "Your Name" + git am ../patches-api/*.patch + + - name: Build and push alpine docker image with API only + id: docker_build_new_api + uses: docker/build-push-action@v2 + with: + context: ./ + file: ./Dockerfile.new + build-args: | + release=1 + add_build_args=-Dapi_only + platforms: linux/amd64,linux/arm64/v8 + push: true + tags: unixfox/invidious-custom:api-new-latest, unixfox/invidious-custom:api-new-build-${{ env.commitid }} + + - name: Build and push official crystal docker image + id: docker_build + uses: docker/build-push-action@v2 + with: + context: ./ + build-args: | + release=1 + file: ./Dockerfile + push: true + tags: unixfox/invidious-custom:latest, unixfox/invidious-custom:build-${{ env.commitid }}