feat(videos): implement video rendition management and cleanup tasks

- Added new environment variables for managing video processing intervals and retention. - Enhanced video rendition creation to prevent duplicates based on profile and output. - Introduced a sweeper task for cleaning up stale video resources. - Updated Docker configuration to include new environment variables and volumes for video processing. - Improved error handling for video states, including handling of aborted and error statuses. - Added tests to ensure proper functionality of new features and error handling.
shimonewman · Jun 16, 2022 · Jun 19, 2022 · Jun 19, 2022 · Jun 20, 2022 · Jun 16, 2022
commit 9d4a95a0c5ab8775c55973ba7e51fd4d6b2319ab
diff --git a/.env b/.env
@@ -148,6 +148,9 @@ _APP_STATS_RESOURCES_QUEUE_NAME=v1-stats-resources
 _APP_STATS_RESOURCES_INTERVAL=3600
 _APP_INTERVAL_CLEANUP_STALE_EXECUTIONS=300
 _APP_INTERVAL_DOMAIN_VERIFICATION=120
+_APP_VIDEOS_STUCK_SWEEP_INTERVAL=300
+_APP_VIDEOS_DOWNLOAD_STUCK_RETENTION=300
+_APP_VIDEOS_ENCODE_STUCK_RETENTION=1800
 _APP_GRAPHQL_INTROSPECTION=enabled
 _APP_GRAPHQL_MAX_BATCH_SIZE=10
 _APP_GRAPHQL_MAX_COMPLEXITY=250

diff --git a/Dockerfile b/Dockerfile
@@ -9,7 +9,12 @@ WORKDIR /usr/local/src/
 COPY composer.lock /usr/local/src/
 COPY composer.json /usr/local/src/
 
-RUN composer install --ignore-platform-reqs --optimize-autoloader \
+RUN apk add --no-cache git openssh-client \
+    && mkdir -p -m 0700 /root/.ssh \
+    && ssh-keyscan github.com >> /root/.ssh/known_hosts
+
+RUN --mount=type=ssh composer install --ignore-platform-reqs --optimize-autoloader \
+        --prefer-source \
         --no-plugins --no-scripts \
         `if [ "$TESTING" != "true" ]; then echo "--no-dev"; fi`
 

diff --git a/app/config/collections/projects.php b/app/config/collections/projects.php
@@ -3289,6 +3289,13 @@
                 'lengths' => [],
                 'orders' => [],
             ],
+            [
+                '$id' => ID::custom('_key_status_updatedAt'),
+                'type' => Database::INDEX_KEY,
+                'attributes' => ['status', '$updatedAt'],
+                'lengths' => [],
+                'orders' => [],
+            ],
         ],
     ],
 
@@ -3580,7 +3587,7 @@
         'indexes' => [
             [
                 '$id' => ID::custom('_key_video_output_profile'),
-                'type' => Database::INDEX_KEY,
+                'type' => Database::INDEX_UNIQUE,
                 'attributes' => ['videoId', 'output', 'profileId'],
                 'lengths' => [],
                 'orders' => [],
@@ -3599,6 +3606,13 @@
                 'lengths' => [],
                 'orders' => [],
             ],
+            [
+                '$id' => ID::custom('_key_status_updatedAt'),
+                'type' => Database::INDEX_KEY,
+                'attributes' => ['status', '$updatedAt'],
+                'lengths' => [],
+                'orders' => [],
+            ],
         ],
     ],
 

diff --git a/app/config/errors.php b/app/config/errors.php
@@ -1601,6 +1601,11 @@
         'description' => 'Video rendition with the requested ID could not be found.',
         'code' => 404,
     ],
+    Exception::VIDEO_RENDITION_ALREADY_EXISTS => [
+        'name' => Exception::VIDEO_RENDITION_ALREADY_EXISTS,
+        'description' => 'A rendition with the same profile and output already exists for this video. Delete it before creating again.',
+        'code' => 409,
+    ],
     Exception::VIDEO_RENDITION_SEGMENT_NOT_FOUND => [
         'name' => Exception::VIDEO_RENDITION_SEGMENT_NOT_FOUND,
         'description' => 'Video rendition segment with the requested ID could not be found.',

diff --git a/bin/clean-stale-videos-resources b/bin/clean-stale-videos-resources
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exec php /usr/src/code/app/cli.php clean-stale-videos-resources "$@"
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -206,6 +206,9 @@ services:
       - _APP_MAINTENANCE_RETENTION_AUDIT_CONSOLE
       - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
       - _APP_MAINTENANCE_RETENTION_SCHEDULES
+      - _APP_VIDEOS_STUCK_SWEEP_INTERVAL
+      - _APP_VIDEOS_DOWNLOAD_STUCK_RETENTION
+      - _APP_VIDEOS_ENCODE_STUCK_RETENTION
       - _APP_SMS_PROVIDER
       - _APP_SMS_FROM
       - _APP_GRAPHQL_INTROSPECTION
@@ -1455,6 +1458,8 @@ services:
     depends_on:
       - ${_APP_DB_HOST:-postgresql}
       - redis
+    volumes:
+      - appwrite-videos-tmp:/storage/videos-tmp:rw
     environment:
       - _APP_ENV
       - _APP_LOGGING_FORMAT
@@ -1482,6 +1487,9 @@ services:
       - _APP_DATABASE_SHARED_TABLES
       - _APP_INTERVAL_DOMAIN_VERIFICATION
       - _APP_INTERVAL_CLEANUP_STALE_EXECUTIONS
+      - _APP_VIDEOS_STUCK_SWEEP_INTERVAL
+      - _APP_VIDEOS_DOWNLOAD_STUCK_RETENTION
+      - _APP_VIDEOS_ENCODE_STUCK_RETENTION
 
   appwrite-task-scheduler-functions:
     profiles:

diff --git a/docs/references/videos/create-rendition.md b/docs/references/videos/create-rendition.md
@@ -2,4 +2,6 @@ Request a new rendition of a video, encoded against a video profile and packaged
 
 The working copy must already be `ready`. Call the create-source endpoint first and wait until the video status is `ready`. If the working copy has been released (`removed`), this request fails with `video_source_removed`. Until the source is ready, this request fails with `video_not_ready`.
 
+Each video may have only one rendition per profile and output combination. Creating a duplicate fails with `video_rendition_already_exists`. After an encode fails or is aborted, delete the rendition and create it again. The same profile may still be encoded for different outputs (for example HLS and DASH).
+
 CMAF packs shared fMP4 segments once and exposes both HLS (`/outputs/cmaf/master.m3u8`) and DASH (`/outputs/cmaf/master.mpd`) masters for the same encode.
diff --git a/phpunit.xml b/phpunit.xml
@@ -12,6 +12,15 @@
     <extensions>
         <bootstrap class="Appwrite\Tests\TestHook" />
     </extensions>
+    <!-- Report deprecations only when first-party code triggers them; PHP 8.4+
+         implicit-nullable notices raised vendor-to-vendor are not actionable here. -->
+    <source ignoreIndirectDeprecations="true">
+        <include>
+            <directory>./app</directory>
+            <directory>./src</directory>
+            <directory>./tests</directory>
+        </include>
+    </source>
     <testsuites>
         <testsuite name="unit">
             <directory>./tests/unit</directory>

diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php
@@ -366,6 +366,7 @@ class Exception extends \Exception
     public const string VIDEO_TRACK_NOT_FOUND = 'video_track_not_found';
     public const string VIDEO_PROFILE_NOT_FOUND = 'video_profile_not_found';
     public const string VIDEO_RENDITION_NOT_FOUND = 'video_rendition_not_found';
+    public const string VIDEO_RENDITION_ALREADY_EXISTS = 'video_rendition_already_exists';
     public const string VIDEO_RENDITION_SEGMENT_NOT_FOUND = 'video_rendition_segment_not_found';
     public const string VIDEO_SUBTITLE_NOT_FOUND = 'video_subtitle_not_found';
     public const string VIDEO_SUBTITLE_NOT_VALID = 'video_subtitle_not_valid';

diff --git a/src/Appwrite/Platform/Modules/Videos/Base.php b/src/Appwrite/Platform/Modules/Videos/Base.php
@@ -43,6 +43,7 @@ abstract class Base extends UtopiaAction
     public const STATUS_UPLOADING = 'uploading';
     public const STATUS_READY = 'ready';
     public const STATUS_ERROR = 'error';
+    public const STATUS_ABORTED = 'aborted';
 
     /**
      * Lifecycle of the video working copy on videos-tmp.
@@ -56,6 +57,26 @@ abstract class Base extends UtopiaAction
     public const SOURCE_READY = 'ready';
     public const SOURCE_REMOVED = 'removed';
     public const SOURCE_ERROR = 'error';
+    public const SOURCE_ABORTED = 'aborted';
+
+    /**
+     * Source statuses the sweeper may abort when stale. Extend here later.
+     *
+     * @var list<string>
+     */
+    public const STALE_SOURCE_STATUSES = [
+        self::SOURCE_DOWNLOADING,
+    ];
+
+    /**
+     * Rendition statuses the sweeper may abort when stale. Extend here later
+     * (not uploading yet — segment upload has no DB heartbeats).
+     *
+     * @var list<string>
+     */
+    public const STALE_ENCODE_STATUSES = [
+        self::STATUS_STARTED,
+    ];
 
     /** Root of a video's working directory on the shared videos-tmp volume. */
     public static function tmpPath(string $projectId, string $videoId): string
@@ -97,6 +118,115 @@ public static function canReleaseSource(string $videoStatus, bool $hasInFlightRe
             && !$jobsRemain;
     }
 
+    /**
+     * Whether the sweeper should abort a video source download.
+     *
+     * Requires status in STALE_SOURCE_STATUSES, incomplete chunks, and
+     * `$updatedAt` older than `$cutoff`.
+     */
+    public static function shouldAbortStaleDownload(Document $video, \DateTimeInterface $cutoff): bool
+    {
+        $status = (string) $video->getAttribute('status', '');
+        if (!\in_array($status, self::STALE_SOURCE_STATUSES, true)) {
+            return false;
+        }
+
+        $uploaded = (int) $video->getAttribute('chunksUploaded', 0);
+        $total = (int) $video->getAttribute('chunksTotal', 1);
+        if ($uploaded >= $total) {
+            return false;
+        }
+
+        $updatedAt = $video->getUpdatedAt();
+        if ($updatedAt === null || $updatedAt === '') {
+            return false;
+        }
+
+        return new \DateTime($updatedAt) < $cutoff;
+    }
+
+    /**
+     * Whether the sweeper should abort a rendition encode.
+     *
+     * Requires status in STALE_ENCODE_STATUSES, progress below 100, and
+     * `$updatedAt` older than `$cutoff`.
+     */
+    public static function shouldAbortStaleEncode(Document $rendition, \DateTimeInterface $cutoff): bool
+    {
+        $status = (string) $rendition->getAttribute('status', '');
+        if (!\in_array($status, self::STALE_ENCODE_STATUSES, true)) {
+            return false;
+        }
+
+        if ((int) $rendition->getAttribute('progress', 0) >= 100) {
+            return false;
+        }
+
+        $updatedAt = $rendition->getUpdatedAt();
+        if ($updatedAt === null || $updatedAt === '') {
+            return false;
+        }
+
+        return new \DateTime($updatedAt) < $cutoff;
+    }
+
+    /**
+     * Unlink the working copy and leftover `.part` / `.decoded` files under
+     * videos-tmp for this video. Does not touch permanent videos storage.
+     */
+    public static function releaseTmpSource(string $projectId, string $videoId): void
+    {
+        $sourcePath = self::tmpSourcePath($projectId, $videoId);
+        $root = \rtrim(APP_STORAGE_VIDEOS_TMP, '/') . '/';
+        if (!\str_starts_with($sourcePath, $root)) {
+            return;
+        }
+
+        foreach (\glob($sourcePath . '*') ?: [] as $path) {
+            if (\is_file($path)) {
+                @\unlink($path);
+            }
+        }
+    }
+
+    /**
+     * Best-effort removal of orphaned encode job directories under videos-tmp.
+     */
+    public static function releaseTmpJobs(string $projectId, string $videoId): void
+    {
+        $jobs = self::tmpPath($projectId, $videoId) . '/jobs';
+        $root = \rtrim(APP_STORAGE_VIDEOS_TMP, '/') . '/';
+        if (!\str_starts_with($jobs, $root) || !\is_dir($jobs)) {
+            return;
+        }
+
+        foreach (\glob($jobs . '/*', GLOB_ONLYDIR) ?: [] as $dir) {
+            self::rmdirRecursive($dir);
+        }
+    }
+
+    private static function rmdirRecursive(string $dir): void
+    {
+        $entries = \scandir($dir);
+        if ($entries === false) {
+            return;
+        }
+
+        foreach ($entries as $entry) {
+            if ($entry === '.' || $entry === '..') {
+                continue;
+            }
+            $path = $dir . '/' . $entry;
+            if (\is_dir($path)) {
+                self::rmdirRecursive($path);
+            } else {
+                @\unlink($path);
+            }
+        }
+
+        @\rmdir($dir);
+    }
+
     /**
      * Timeline and rendition creates require a live working copy.
      */

diff --git a/src/Appwrite/Platform/Modules/Videos/Http/Videos/Renditions/Create.php b/src/Appwrite/Platform/Modules/Videos/Http/Videos/Renditions/Create.php
@@ -16,6 +16,7 @@
 use Utopia\Database\Database;
 use Utopia\Database\Document;
 use Utopia\Database\Helpers\ID;
+use Utopia\Database\Query;
 use Utopia\Database\Validator\Authorization;
 use Utopia\Database\Validator\UID;
 use Utopia\Platform\Action;
@@ -93,6 +94,16 @@ public function action(
 
         $this->assertSourceReady($video);
 
+        $existing = $authorization->skip(fn () => $dbForProject->find('videos_renditions', [
+            Query::equal('videoInternalId', [$video->getSequence()]),
+            Query::equal('profileId', [$profile->getId()]),
+            Query::equal('output', [$output]),
+            Query::limit(1),
+        ]));
+        if (!empty($existing)) {
+            throw new Exception(Exception::VIDEO_RENDITION_ALREADY_EXISTS);
+        }
+
         $width = (int) $profile->getAttribute('width', 0);
         $height = (int) $profile->getAttribute('height', 0);
         $videoBitRate = (int) $profile->getAttribute('videoBitRate', 0);

diff --git a/src/Appwrite/Platform/Modules/Videos/Http/Videos/Renditions/XList.php b/src/Appwrite/Platform/Modules/Videos/Http/Videos/Renditions/XList.php
@@ -53,7 +53,7 @@ public function __construct()
             ))
             ->param('videoId', '', new UID(), 'Video unique ID.')
             ->param('output', null, new Nullable(new WhiteList(self::OUTPUTS, true)), 'Only return renditions packaged for this output format.', true, enum: new Enum(name: 'VideoOutput'))
-            ->param('status', null, new Nullable(new WhiteList([self::STATUS_WAITING, self::STATUS_STARTED, self::STATUS_ENDED, self::STATUS_UPLOADING, self::STATUS_READY, self::STATUS_ERROR], true)), 'Only return renditions in this transcoding state.', true, enum: new Enum(name: 'VideoRenditionStatus'))
+            ->param('status', null, new Nullable(new WhiteList([self::STATUS_WAITING, self::STATUS_STARTED, self::STATUS_ENDED, self::STATUS_UPLOADING, self::STATUS_READY, self::STATUS_ERROR, self::STATUS_ABORTED], true)), 'Only return renditions in this transcoding state.', true, enum: new Enum(name: 'VideoRenditionStatus'))
             ->inject('response')
             ->inject('dbForProject')
             ->inject('user')

diff --git a/src/Appwrite/Platform/Modules/Videos/Http/Videos/Source/Create.php b/src/Appwrite/Platform/Modules/Videos/Http/Videos/Source/Create.php
@@ -81,7 +81,8 @@ public function action(
 
         // A live or in-flight working copy must not be fetched twice — reject
         // explicitly rather than no-op so the caller knows the request did
-        // nothing. `pending`, `removed` and `error` fall through to enqueue.
+        // nothing. `pending`, `removed`, `error` and `aborted` fall through
+        // to enqueue (re-download after a stale-download sweep).
         if ($status === self::SOURCE_DOWNLOADING) {
             throw new Exception(Exception::VIDEO_SOURCE_IN_PROGRESS);
         }

diff --git a/src/Appwrite/Platform/Modules/Videos/Module.php b/src/Appwrite/Platform/Modules/Videos/Module.php
@@ -3,6 +3,7 @@
 namespace Appwrite\Platform\Modules\Videos;
 
 use Appwrite\Platform\Modules\Videos\Services\Http;
+use Appwrite\Platform\Modules\Videos\Services\Tasks;
 use Appwrite\Platform\Modules\Videos\Services\Workers;
 use Utopia\Platform;
 
@@ -12,5 +13,6 @@ public function __construct()
     {
         $this->addService('http', new Http());
         $this->addService('workers', new Workers());
+        $this->addService('tasks', new Tasks());
     }
 }
diff --git a/src/Appwrite/Platform/Modules/Videos/Services/Tasks.php b/src/Appwrite/Platform/Modules/Videos/Services/Tasks.php
@@ -0,0 +1,15 @@
+<?php
+
+namespace Appwrite\Platform\Modules\Videos\Services;
+
+use Appwrite\Platform\Modules\Videos\Tasks\CleanStaleVideosResources;
+use Utopia\Platform\Service;
+
+class Tasks extends Service
+{
+    public function __construct()
+    {
+        $this->type = Service::TYPE_TASK;
+        $this->addAction(CleanStaleVideosResources::getName(), new CleanStaleVideosResources());
+    }
+}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		#!/bin/sh

		exec php /usr/src/code/app/cli.php clean-stale-videos-resources "$@"
Original file line number	Diff line number	Diff line change
Expand Up		@@ -2,4 +2,6 @@ Request a new rendition of a video, encoded against a video profile and packaged

		The working copy must already be `ready`. Call the create-source endpoint first and wait until the video status is `ready`. If the working copy has been released (`removed`), this request fails with `video_source_removed`. Until the source is ready, this request fails with `video_not_ready`.

		Each video may have only one rendition per profile and output combination. Creating a duplicate fails with `video_rendition_already_exists`. After an encode fails or is aborted, delete the rendition and create it again. The same profile may still be encoded for different outputs (for example HLS and DASH).

		CMAF packs shared fMP4 segments once and exposes both HLS (`/outputs/cmaf/master.m3u8`) and DASH (`/outputs/cmaf/master.mpd`) masters for the same encode.