diff --git a/.gitea/workflows/work.yml b/.gitea/workflows/work.yml new file mode 100644 index 0000000..61ac7a6 --- /dev/null +++ b/.gitea/workflows/work.yml @@ -0,0 +1,75 @@ +name: Build and Deploy on Tag +on: + push: + tags: + - "v*" + +env: + PACKAGE_AND_BREW: ${{ secrets.PACKAGE_AND_BREW }} + VERSION: ${{ gitea.ref_name }} + GITEA_DOMAIN: git.tesses.org + GITEA_REGISTRY_USER: tesses50 + RESULT_IMAGE_NAME: tesses50/tytd2025 + +jobs: + build-ppc: + name: "🔨 Build for PowerPC" + runs-on: ubuntu-latest + container: + image: git.tesses.org/tesses50/linux-ppc:latest + steps: + - uses: actions/checkout@v4 + - run: bash build-for-ppc.sh + - uses: akkuman/gitea-release-action@v1 + env: + NODE_OPTIONS: '--experimental-fetch' # if nodejs < 18 + with: + prerelease: true + files: |- + artifacts/** + + build-packages: + name: "🔨 Build for everything else" + runs-on: fw13-ubuntu-latest + + container: + image: "git.tesses.org/tesses50/linux-x64:latest" + steps: + - uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Log in to registry + uses: docker/login-action@v3 + with: + registry: ${{ env.GITEA_DOMAIN }} + username: ${{ env.GITEA_REGISTRY_USER }} + password: ${{ secrets.PACKAGE_AND_BREW }} + - name: Build and push image + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + push: true + tags: ${{ env.GITEA_DOMAIN }}/${{ env.RESULT_IMAGE_NAME }}:latest + - name: Install CrossLang + uses: https://git.tesses.org/tesses50/setup-crosslang@v1 + - run: | + bash build-for-system-pkg.sh + cd Tesses.YouTubeDownloader + crosslang upload-package --token="${{ secrets.CPKG_KEY }}" --host="https://cpkg.tesseslanguage.com/" + cd ../Tesses.YouTubeDownloader.Server + crosslang upload-package --token="${{ secrets.CPKG_KEY }}" --host="https://cpkg.tesseslanguage.com/" + cd ../Tesses.YouTubeDownloader.PluginTemplate + crosslang upload-package --token="${{ secrets.CPKG_KEY }}" --host="https://cpkg.tesseslanguage.com/" + + - uses: akkuman/gitea-release-action@v1 + env: + NODE_OPTIONS: '--experimental-fetch' # if nodejs < 18 + with: + prerelease: true + files: |- + artifacts/** + + + + \ No newline at end of file diff --git a/.gitignore b/.gitignore index 6e474a0..4ac87b8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,8 @@ bin obj publish -TYTD \ No newline at end of file +TYTD +artifacts +*.pkg.tar.zst +*.deb +*.rpm \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 21eb1ab..0372c5d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM onedev.site.tesses.net/crosslang/crosslang:latest +FROM git.tesses.org/tesses50/crosslang-withshell:latest RUN mkdir /app COPY Tesses.YouTubeDownloader.Server/bin/ /app WORKDIR /data diff --git a/Tesses.YouTubeDownloader.PluginTemplate/cross.json b/Tesses.YouTubeDownloader.PluginTemplate/cross.json index 1430446..4b2b323 100644 --- a/Tesses.YouTubeDownloader.PluginTemplate/cross.json +++ b/Tesses.YouTubeDownloader.PluginTemplate/cross.json @@ -28,6 +28,6 @@ "template_icon": "icon.png" }, "name": "Tesses.YouTubeDownloader.PluginTemplate", - "version": "1.0.0.0-prod", + "version": "1.0.0.0-dev", "icon": "icon.png" } \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Server/cross.json b/Tesses.YouTubeDownloader.Server/cross.json index 82d8e17..af5e9d6 100644 --- a/Tesses.YouTubeDownloader.Server/cross.json +++ b/Tesses.YouTubeDownloader.Server/cross.json @@ -14,6 +14,6 @@ "project_dependencies": [ "..\/Tesses.YouTubeDownloader" ], - "version": "1.0.0.6-dev", + "version": "1.0.0.0-dev", "compTime": "secure" } \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Server/res/offline.html b/Tesses.YouTubeDownloader.Server/res/offline.html index 89d0294..62942e4 100644 --- a/Tesses.YouTubeDownloader.Server/res/offline.html +++ b/Tesses.YouTubeDownloader.Server/res/offline.html @@ -12,6 +12,10 @@
+ + home + Home +
diff --git a/Tesses.YouTubeDownloader.Server/res/service_worker.js b/Tesses.YouTubeDownloader.Server/res/service_worker.js index 1d32a87..3c69cd7 100644 --- a/Tesses.YouTubeDownloader.Server/res/service_worker.js +++ b/Tesses.YouTubeDownloader.Server/res/service_worker.js @@ -2,6 +2,13 @@ const assets = ["<@ASSETS@>"]; const staticCacheName = "tytd-static-<@BUILD_TIME@>"; +async function ThrowOnBadGateway(req) +{ + const resp = await fetch(req); + if(resp.status === 502) throw new Error("Bad Gateway"); + return resp; +} + self.addEventListener('install',evt => { evt.waitUntil( caches.open(staticCacheName).then(cache =>{ @@ -19,24 +26,29 @@ self.addEventListener('activate',(evt)=>{ }); self.addEventListener('fetch', evt => { - const uri = new URL(evt.request.url); evt.respondWith( - + caches.match(evt.request).then(cacheRes=>{ - return cacheRes || fetch(evt.request); - }).catch(()=>{ - if(uri.pathname === '/queue-size') + console.warn('SW cacheRes:', cacheRes); + return cacheRes || ThrowOnBadGateway(evt.request); + }).catch((err)=>{ + console.warn('SW Fallback:', err.message); + if(evt.request.headers.has('HX-Request')) { - return new Response('?',{ - headers: { 'Content-Type': 'text/html' } - }); + + return new Response("",{ + status: 200, + headers: { + 'HX-Redirect': "/offline.html" + } + }); } - if(uri.pathname === "/progress") + else { - return caches.match("/offline-progress.html"); + return caches.match('/offline.html'); + } - return caches.match('/offline.html'); }) ); -}); \ No newline at end of file +}); diff --git a/Tesses.YouTubeDownloader.Server/src/components/discover_entry.tcross b/Tesses.YouTubeDownloader.Server/src/components/discover_entry.tcross deleted file mode 100644 index 59d2893..0000000 --- a/Tesses.YouTubeDownloader.Server/src/components/discover_entry.tcross +++ /dev/null @@ -1,62 +0,0 @@ -func Components.DiscoverEntry(item) -{ - switch(item.type) - { - case "video": - return -
-
- -
-
-
- -
- {item.views} • {item.uploaded} -
-
- - - - {part.text} - - - - {part.text} - - - -
- -
-
-
; - break; - case "channel": - return -
-
- -
-
-
- -
- {item.subs} -
-
-

{item.description}

-
-
- -
-
; - break; - } - - return ""; -} \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Server/src/components/music-album-entry.tcross b/Tesses.YouTubeDownloader.Server/src/components/music-album-entry.tcross deleted file mode 100644 index 58c779c..0000000 --- a/Tesses.YouTubeDownloader.Server/src/components/music-album-entry.tcross +++ /dev/null @@ -1,11 +0,0 @@ -func Components.MusicAlbum(item) -{ - return
-
- -
- -
; -} \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Server/src/components/music-artist.tcross b/Tesses.YouTubeDownloader.Server/src/components/music-artist.tcross deleted file mode 100644 index d62e640..0000000 --- a/Tesses.YouTubeDownloader.Server/src/components/music-artist.tcross +++ /dev/null @@ -1,4 +0,0 @@ -func Components.MusicArtist(item) -{ - return {item.name}; -} \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Server/src/components/queuesz.tcross b/Tesses.YouTubeDownloader.Server/src/components/queuesz.tcross index d093777..0755061 100644 --- a/Tesses.YouTubeDownloader.Server/src/components/queuesz.tcross +++ b/Tesses.YouTubeDownloader.Server/src/components/queuesz.tcross @@ -5,7 +5,7 @@ func Components.QueueSZ(tytd) tytd.Mutex.Lock(); - var html = {tytd.VideoQueueCount}; + var html = {tytd.VideoQueueCount}; tytd.Mutex.Unlock(); return html; diff --git a/Tesses.YouTubeDownloader.Server/src/components/shell.tcross b/Tesses.YouTubeDownloader.Server/src/components/shell.tcross index 0bee664..d86ed39 100644 --- a/Tesses.YouTubeDownloader.Server/src/components/shell.tcross +++ b/Tesses.YouTubeDownloader.Server/src/components/shell.tcross @@ -21,34 +21,48 @@ func Components.Shell(title, html, page, $mypage) text="Home", icon="home", href=index, - classStr="" + classStr="", + badge = "" + }, + { + text="Queue", + icon="downloading", + href=(/"queue").MakeRelative(mypage).ToString(),, + classStr="", + badge= + ? + }, { text = "Downloads", icon = "download", href=(/"downloads").MakeRelative(mypage).ToString(), - classStr="" + classStr="", + badge="" }, { text = "Plugins", icon = "extension", href= (/"plugins").MakeRelative(mypage).ToString(), - classStr="" + classStr="", + badge="" }, { text = "Settings", icon = "settings", href= (/"settings").MakeRelative(mypage).ToString(), - classStr="" + classStr="", + badge="" }, { text = "Api", icon = "api", href = (/"api").MakeRelative(mypage).ToString(), - classStr="" + classStr="", + badge="" } ]; - pages[page].classStr = "active"; + pages[page].classStr = " primary-container"; return @@ -65,8 +79,43 @@ func Components.Shell(title, html, page, $mypage) - -*/
diff --git a/Tesses.YouTubeDownloader.Server/src/main.tcross b/Tesses.YouTubeDownloader.Server/src/main.tcross index a5984ff..caff570 100644 --- a/Tesses.YouTubeDownloader.Server/src/main.tcross +++ b/Tesses.YouTubeDownloader.Server/src/main.tcross @@ -58,6 +58,11 @@ class TYTDApp { } public Handle(ctx) { + if(ctx.Path == "/api/v1/resolutions.json") + { + ctx.SendJson(Resolution.Resolutions); + return true; + } if(ctx.Path == "/service_worker.js" || fileNames.Contains(ctx.Path)) { const inm=ctx.RequestHeaders.TryGetFirst("If-None-Match"); @@ -176,6 +181,10 @@ class TYTDApp { ); return true; } + if(ctx.Path == "/queue-size") + { + return false; + } if(ctx.Path == "/sso") { const app = ctx.QueryParams.TryGetFirst("app") ?? ""; @@ -570,6 +579,21 @@ class TYTDApp { ctx.WithMimeType("application/json").SendJson(this.TYTD.GetChannelContents(id,page-1,count)); return true; } + else if(ctx.Path == "/api/v1/queue.json") + { + if(ctx.Method == "GET") + { + var page = ctx.QueryParams.TryGetFirstInt("page"); + var count = ctx.QueryParams.TryGetFirstInt("count"); + if(TypeOf(page)!="Long") page = 1; + if(TypeOf(count)!="Long") count = 20; + + + var result = { entries = TYTD.GetQueueItems(page-1,count)}; + ctx.WithMimeType("application/json").SendJson(result); + return true; + } + } else if(ctx.Path == "/api/v1/downloads.json") { var q = ctx.QueryParams.TryGetFirst("q"); @@ -605,7 +629,7 @@ class TYTDApp { if(!this.TYTD.SendDatabase(ctx)) { - ctx.WithMimeType("text/html").SendText(Components.Shell("Unauthorized","

You are not authorized to download the database

",3,/"api/v1/database.db")); + ctx.WithMimeType("text/html").SendText(Components.Shell("Unauthorized","

You are not authorized to download the database

",4,/"api/v1/database.db")); } return true; } @@ -773,6 +797,24 @@ class TYTDApp { return true; } } + else if(ctx.Path == "/queue") + { + + if(ctx.Method == "DELETE") + { + const id = ctx.QueryParams.TryGetFirstInt("id"); + if(TypeIsLong(id)) + { + this.TYTD.QueueRemoveItem(id); + } + else + { + this.TYTD.QueueClear(); + } + } + ctx.WithMimeType("text/html").SendText(Pages.Queue(this.TYTD,ctx)); + return true; + } else if(ctx.Path == "/downloads") { ctx.WithMimeType("text/html").SendText(Pages.Downloads(this.TYTD,ctx)); @@ -793,7 +835,7 @@ class TYTDApp { { if(!UserFlags.IsAdmin(this.TYTD.IsLoggedIn(ctx))) { - ctx.WithMimeType("text/html").SendText(Components.Shell("Unauthorized",

You can{"'"}t modify admin settings as you are not an admin

,3)); + ctx.WithMimeType("text/html").SendText(Components.Shell("Unauthorized",

You can{"'"}t modify admin settings as you are not an admin

,4)); return true; } const user = ctx.QueryParams.TryGetFirst("user"); @@ -871,7 +913,7 @@ class TYTDApp { { if(!UserFlags.IsAdmin(this.TYTD.IsLoggedIn(ctx))) { - ctx.WithMimeType("text/html").SendText(Components.Shell("Unauthorized",

You can{"'"}t modify admin settings as you are not an admin

,3)); + ctx.WithMimeType("text/html").SendText(Components.Shell("Unauthorized",

You can{"'"}t modify admin settings as you are not an admin

,4)); return true; } if(ctx.Method == "POST") diff --git a/Tesses.YouTubeDownloader.Server/src/pages/admin.tcross b/Tesses.YouTubeDownloader.Server/src/pages/admin.tcross index ab69271..d5694d8 100644 --- a/Tesses.YouTubeDownloader.Server/src/pages/admin.tcross +++ b/Tesses.YouTubeDownloader.Server/src/pages/admin.tcross @@ -72,5 +72,5 @@ func Pages.Admin(tytd,ctx) ; - return Components.Shell("Admin Settings",html ,3); + return Components.Shell("Admin Settings",html ,4); } \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Server/src/pages/api.tcross b/Tesses.YouTubeDownloader.Server/src/pages/api.tcross index 8cdbe40..9607355 100644 --- a/Tesses.YouTubeDownloader.Server/src/pages/api.tcross +++ b/Tesses.YouTubeDownloader.Server/src/pages/api.tcross @@ -8,5 +8,5 @@ func Pages.Api() ; - return Components.Shell("Api",html,4); + return Components.Shell("Api",html,5); } \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Server/src/pages/api/api-v1.tcross b/Tesses.YouTubeDownloader.Server/src/pages/api/api-v1.tcross index 80b147a..1375cfd 100644 --- a/Tesses.YouTubeDownloader.Server/src/pages/api/api-v1.tcross +++ b/Tesses.YouTubeDownloader.Server/src/pages/api/api-v1.tcross @@ -21,7 +21,7 @@ func Pages.ApiV1() ; - return Components.Shell("Api V1",html,4); + return Components.Shell("Api V1",html,5); } var apiv1_routes = Json.Decode(embed("apiv1_routes.json").ToString()); diff --git a/Tesses.YouTubeDownloader.Server/src/pages/changepassword.tcross b/Tesses.YouTubeDownloader.Server/src/pages/changepassword.tcross index 5daf7d9..bb95c0e 100644 --- a/Tesses.YouTubeDownloader.Server/src/pages/changepassword.tcross +++ b/Tesses.YouTubeDownloader.Server/src/pages/changepassword.tcross @@ -45,5 +45,5 @@ const html = ; - return Components.Shell("Change your password",html,3); + return Components.Shell("Change your password",html,4); } \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Server/src/pages/channel/info.tcross b/Tesses.YouTubeDownloader.Server/src/pages/channel/info.tcross index 12ec196..4049c44 100644 --- a/Tesses.YouTubeDownloader.Server/src/pages/channel/info.tcross +++ b/Tesses.YouTubeDownloader.Server/src/pages/channel/info.tcross @@ -35,5 +35,5 @@ func Pages.ChannelInfo(tytd,ctx) title = res.authorName; } - return Components.Shell($"Channel {title}",html,1); + return Components.Shell($"Channel {title}",html,2); } \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Server/src/pages/discover.tcross b/Tesses.YouTubeDownloader.Server/src/pages/discover.tcross deleted file mode 100644 index c05faf2..0000000 --- a/Tesses.YouTubeDownloader.Server/src/pages/discover.tcross +++ /dev/null @@ -1,34 +0,0 @@ -func Pages.Discover(tytd,ctx) -{ - - var q = ctx.QueryParams.TryGetFirst("q"); - var res = null; - var q2 = ""; - if(TypeOf(q) == "String") - { - q2 = q; - res = tytd.Discover(q); - - } - - var html = -
-
- search - - -
-
- - -
    - -
  • - -
- -
; - - - return Components.Shell($"Discover {q2}",html,1); -} \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Server/src/pages/downloads.tcross b/Tesses.YouTubeDownloader.Server/src/pages/downloads.tcross index 15cd72c..83e6892 100644 --- a/Tesses.YouTubeDownloader.Server/src/pages/downloads.tcross +++ b/Tesses.YouTubeDownloader.Server/src/pages/downloads.tcross @@ -289,5 +289,5 @@ func Pages.Downloads(tytd,ctx) - return Components.Shell($"Downloads {q}",html,1); + return Components.Shell($"Downloads {q}",html,2); } \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Server/src/pages/edituser.tcross b/Tesses.YouTubeDownloader.Server/src/pages/edituser.tcross index d0df2ad..03f3d5e 100644 --- a/Tesses.YouTubeDownloader.Server/src/pages/edituser.tcross +++ b/Tesses.YouTubeDownloader.Server/src/pages/edituser.tcross @@ -38,7 +38,7 @@ func Pages.EditUserList(tytd, ctx) ; - return Components.Shell("Edit users",html, 3); + return Components.Shell("Edit users",html, 4); } func Pages.EditUser(user) @@ -141,5 +141,5 @@ func Pages.EditUser(user) ; - return Components.Shell("Edit user",html, 3); + return Components.Shell("Edit user",html, 4); } \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Server/src/pages/list.tcross b/Tesses.YouTubeDownloader.Server/src/pages/list.tcross index bb2659b..1c0d578 100644 --- a/Tesses.YouTubeDownloader.Server/src/pages/list.tcross +++ b/Tesses.YouTubeDownloader.Server/src/pages/list.tcross @@ -35,5 +35,5 @@ func Pages.List(tytd,ctx) ; - return Components.Shell($"Personal List {name}",html,1); + return Components.Shell($"Personal List {name}",html,2); } \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Server/src/pages/music/artist.tcross b/Tesses.YouTubeDownloader.Server/src/pages/music/artist.tcross deleted file mode 100644 index 2107ed6..0000000 --- a/Tesses.YouTubeDownloader.Server/src/pages/music/artist.tcross +++ /dev/null @@ -1,25 +0,0 @@ -func Pages.Artist(music,ctx) -{ - - var id = ctx.QueryParams.TryGetFirst("id"); - var res = music.GetArtistsAlbums(id); - - if(TypeOf(id) != "String") - { - return Components.Shell("Error",

Artist id not specified

,pages); - } - - - var html = -

{res.name}

-
    - -
  • - -
-
; - - - return Components.Shell($"Artist {res.name}",html,2); - -} \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Server/src/pages/music/artists.tcross b/Tesses.YouTubeDownloader.Server/src/pages/music/artists.tcross deleted file mode 100644 index 100fadf..0000000 --- a/Tesses.YouTubeDownloader.Server/src/pages/music/artists.tcross +++ /dev/null @@ -1,33 +0,0 @@ -func Pages.Artists(music,ctx) -{ - - var q = ctx.QueryParams.TryGetFirst("q"); - var res = null; - var q2 = ""; - if(TypeOf(q) == "String") - { - q2 = q; - res = music.GetArtists(q); - - } - - var html = -
-
- search - - -
-
- - -
    - -
  • - -
-
; - - - return Components.Shell($"Search Artists {q2}",html,2); -} \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Server/src/pages/music/music.tcross b/Tesses.YouTubeDownloader.Server/src/pages/music/music.tcross deleted file mode 100644 index 4da2516..0000000 --- a/Tesses.YouTubeDownloader.Server/src/pages/music/music.tcross +++ /dev/null @@ -1,8 +0,0 @@ -func Pages.Music(ctx) -{ - var html = ; - return Components.Shell("Music",html,2); -} \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Server/src/pages/newaccount.tcross b/Tesses.YouTubeDownloader.Server/src/pages/newaccount.tcross index 37aba13..9da9521 100644 --- a/Tesses.YouTubeDownloader.Server/src/pages/newaccount.tcross +++ b/Tesses.YouTubeDownloader.Server/src/pages/newaccount.tcross @@ -81,5 +81,5 @@ func Pages.NewUser(error) ; - return Components.Shell("New User",html ,3); + return Components.Shell("New User",html ,4); } \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Server/src/pages/personal-from-yt-temp.tcross b/Tesses.YouTubeDownloader.Server/src/pages/personal-from-yt-temp.tcross index 710421c..33ad237 100644 --- a/Tesses.YouTubeDownloader.Server/src/pages/personal-from-yt-temp.tcross +++ b/Tesses.YouTubeDownloader.Server/src/pages/personal-from-yt-temp.tcross @@ -22,5 +22,5 @@ func Pages.YouTubeAnonyPlaylist(video_ids)
; - return Components.Shell("Create playlist",html ,1); + return Components.Shell("Create playlist",html ,2); } \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Server/src/pages/playlist/info.tcross b/Tesses.YouTubeDownloader.Server/src/pages/playlist/info.tcross index 61b8408..b7c73c6 100644 --- a/Tesses.YouTubeDownloader.Server/src/pages/playlist/info.tcross +++ b/Tesses.YouTubeDownloader.Server/src/pages/playlist/info.tcross @@ -35,5 +35,5 @@ func Pages.PlaylistInfo(tytd,ctx) title = res.authorName; } - return Components.Shell($"Playlist {title}",html,1); + return Components.Shell($"Playlist {title}",html,2); } \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Server/src/pages/plugins-dl.tcross b/Tesses.YouTubeDownloader.Server/src/pages/plugins-dl.tcross index 5c03a2f..73ddcb9 100644 --- a/Tesses.YouTubeDownloader.Server/src/pages/plugins-dl.tcross +++ b/Tesses.YouTubeDownloader.Server/src/pages/plugins-dl.tcross @@ -75,5 +75,5 @@ func Pages.DownloadPlugins(tytd,ctx) ; - return Components.Shell("Download plugins",html ,2); + return Components.Shell("Download plugins",html ,3); } \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Server/src/pages/plugins.tcross b/Tesses.YouTubeDownloader.Server/src/pages/plugins.tcross index 7ff683c..b3eef41 100644 --- a/Tesses.YouTubeDownloader.Server/src/pages/plugins.tcross +++ b/Tesses.YouTubeDownloader.Server/src/pages/plugins.tcross @@ -20,5 +20,5 @@ func Pages.Plugins(tytd,ctx) ; - return Components.Shell("Installed plugins", UserFlags.CanUsePlugins(userFlags) ? html :

You can{"'"}t use plugins

,2); + return Components.Shell("Installed plugins", UserFlags.CanUsePlugins(userFlags) ? html :

You can{"'"}t use plugins

,3); } \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Server/src/pages/queue.tcross b/Tesses.YouTubeDownloader.Server/src/pages/queue.tcross new file mode 100644 index 0000000..0a18ca6 --- /dev/null +++ b/Tesses.YouTubeDownloader.Server/src/pages/queue.tcross @@ -0,0 +1,71 @@ + +func Pages.Queue(tytd,ctx) +{ + var page = ctx.QueryParams.TryGetFirstInt("page"); + if(TypeOf(page) != "Long") page = 1; + page--; + + + enumerable func getitems() + { + each(var item : tytd.GetQueueItems(page,10)) + { + tytd.PutVideoInfoIfNotExists(item.videoId); + const info = tytd.GetVideo(item.videoId); + + yield { + Title = $"{info.title}-{item.videoId}-{item.resolution}", + VideoId = item.videoId, + Id = item.id, + ChannelId = info.channelId, + Channel = info.author + }; + } + } + + + const html = +
+
+ +
+
+ +
+
+ + + +
+
+ +
+
+
+
+ {item.Title} +
+ +
+ {item.Channel} + + +
+ +
+
+
+ +
+
+ + +
+ + {page+1} + +
+ +
; + return Components.Shell("Queue",html,1); +} \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Server/src/pages/settings.tcross b/Tesses.YouTubeDownloader.Server/src/pages/settings.tcross index aa9d7a6..e34589a 100644 --- a/Tesses.YouTubeDownloader.Server/src/pages/settings.tcross +++ b/Tesses.YouTubeDownloader.Server/src/pages/settings.tcross @@ -15,5 +15,5 @@ func Pages.Settings(tytd,ctx) ; - return Components.Shell("Settings",html ,3); + return Components.Shell("Settings",html ,4); } \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Server/src/pages/video/info.tcross b/Tesses.YouTubeDownloader.Server/src/pages/video/info.tcross index 0561be0..1412a17 100644 --- a/Tesses.YouTubeDownloader.Server/src/pages/video/info.tcross +++ b/Tesses.YouTubeDownloader.Server/src/pages/video/info.tcross @@ -27,5 +27,5 @@ func Pages.VideoInfo(tytd,ctx)
; } - return Components.Shell(vi != null ? vi.title : "Could not find video", html, 1); + return Components.Shell(vi != null ? vi.title : "Could not find video", html, 2); } \ No newline at end of file diff --git a/Tesses.YouTubeDownloader.Server/src/pages/whoami.tcross b/Tesses.YouTubeDownloader.Server/src/pages/whoami.tcross index b0102e2..6e2dc55 100644 --- a/Tesses.YouTubeDownloader.Server/src/pages/whoami.tcross +++ b/Tesses.YouTubeDownloader.Server/src/pages/whoami.tcross @@ -11,5 +11,5 @@ func Pages.WhoAmI(row)
  • Can manage plugins: {UserFlags.CanManagePlugins(row.flags)}
  • ; - return Components.Shell("Settings",html ,3); + return Components.Shell("Settings",html ,4); } \ No newline at end of file diff --git a/Tesses.YouTubeDownloader/cross.json b/Tesses.YouTubeDownloader/cross.json index 3aeba32..737909a 100644 --- a/Tesses.YouTubeDownloader/cross.json +++ b/Tesses.YouTubeDownloader/cross.json @@ -14,6 +14,6 @@ } ], "name": "Tesses.YouTubeDownloader", - "version": "1.0.0.6-dev", + "version": "1.0.0.0-dev", "icon": "icon.png" } \ No newline at end of file diff --git a/Tesses.YouTubeDownloader/src/Queue.tcross b/Tesses.YouTubeDownloader/src/Queue.tcross deleted file mode 100644 index fd68bb6..0000000 --- a/Tesses.YouTubeDownloader/src/Queue.tcross +++ /dev/null @@ -1,35 +0,0 @@ -class TYTD.Queue -{ - private ls = []; - private mtx = new Muxex(); - - public getCount() - { - this.mtx.Lock(); - var c = this.ls.Count; - this.mtx.Unlock(); - return c; - } - - public Pop() - { - var item = null; - this.mtx.Lock(); - if(this.ls.Count > 0) - { - item = ls[ls.Count-1]; - this.ls.RemoveAt(ls.Count-1); - } - - - this.mtx.Unlock(); - return item; - } - - public Push(val) - { - this.mtx.Lock(); - this.ls.Add(val); - this.mtx.Unlock(); - } -} \ No newline at end of file diff --git a/Tesses.YouTubeDownloader/src/Resolution.tcross b/Tesses.YouTubeDownloader/src/Resolution.tcross index d2af79d..5d242e8 100644 --- a/Tesses.YouTubeDownloader/src/Resolution.tcross +++ b/Tesses.YouTubeDownloader/src/Resolution.tcross @@ -4,33 +4,21 @@ class Resolution static getNoDownload() "NoDownload"; /^ Get the video/audio mp4 (muxed by YouTube) ^/ static getLowVideo() "LowVideo"; - /^ Get the highest video stream ^/ - static getVideoOnly() "VideoOnly"; - /^ Get the highest audio stream ^/ - static getAudioOnly() "AudioOnly"; - /^ Get the highest audio stream (and convert to mp3) ^/ + /^ Get the LowVideo stream (and convert to mp3) ^/ static getMP3() "MP3"; - /^ Get the highest audio stream (and convert to flac) ^/ + /^ Get the LowVideo stream (and convert to flac) ^/ static getFLAC() "FLAC"; - /^ Get the highest video and then audio stream (and convert to mp4) ^/ - static getMP4() "MP4"; - /^ Get the highest video and then audio stream (and mux to a mkv file) ^/ - static getMKV() "MKV"; - /^ Get the highest video and then audio stream (dont convert or mux) ^/ - static getDontConvert() "DontConvert"; - + /^ Get the LowVideo stream (and convert to AMV for mp4 players like: https://www.ebay.com/sch/i.html?_nkw=AMV+mp3+player) ^/ + static getAMV() "AMV"; + static getResolutions() { return [ - {name="Don't Download", value=Resolution.NoDownload}, - {name="Low (muxed by YouTube)", value=Resolution.LowVideo,default=true}, - {name="Highest video (no audio)", value=Resolution.VideoOnly}, - {name="Highest audio (no video)", value=Resolution.AudioOnly}, - {name="Convert to MP3", value=Resolution.MP3}, - {name="Convert to FLAC", value=Resolution.FLAC}, - {name="Convert to MP4", value=Resolution.MP4}, - {name="Mux to MKV (no transcoding)",value=Resolution.MKV}, - {name="Don't convert or Mux",value=Resolution.DontConvert} + {name="Don't Download", value=Resolution.NoDownload,filename=null}, + {name="Low (muxed by YouTube)", value=Resolution.LowVideo,default=true,filename="ytmux.mp4"}, + {name="Convert to MP3", value=Resolution.MP3,filename="conv.mp3"}, + {name="Convert to FLAC", value=Resolution.FLAC,filename="conv.flac"}, + {name="Convert To AMV (for generic mp4 players)", value=Resolution.AMV,filename="conv.amv"} ]; } } \ No newline at end of file diff --git a/Tesses.YouTubeDownloader/src/YouTubeDownloader.tcross b/Tesses.YouTubeDownloader/src/YouTubeDownloader.tcross index f40405a..d0d0f9f 100644 --- a/Tesses.YouTubeDownloader/src/YouTubeDownloader.tcross +++ b/Tesses.YouTubeDownloader/src/YouTubeDownloader.tcross @@ -67,6 +67,50 @@ class TYTD.Downloader { this.Storage = vfs; this.DatabaseDirectory = dbDir; } + + private PopQueue() + { + Mutex.Lock(); + const db = OpenDB(); + var res = Sqlite.Exec(db,"SELECT * FROM queue ORDER BY id DESC LIMIT 1;"); + + if(TypeIsList(res) && res.Length > 0) + { + Sqlite.Exec(db,$"DELETE FROM queue WHERE id = {Sqlite.Escape(res[0].id)};"); + res = res[0]; + } + else { res = null;} + Sqlite.Close(db); + Mutex.Unlock(); + + + + if(TypeIsDictionary(res)) + { + switch(res.resolution) + { + case Resolution.NoDownload: + { + this.PutVideoInfoIfNotExists(res.videoId); + + return null; + } + + case Resolution.MP3: + return new TYTD.TranscodeAudio(res.videoId,".mp3"); + case Resolution.FLAC: + return new TYTD.TranscodeAudio(res.videoId,".flac"); + case Resolution.AMV: + return new TYTD.TranscodeAMV(res.videoId); + default: + return new TYTD.SDVideoDownload(res.videoId); + + } + } + return null; + } + + /^ Download a video id: video id or url @@ -77,6 +121,8 @@ class TYTD.Downloader { const theVideoId = TYTD.GetVideoId(id); if(TypeIsString(theVideoId)) { + + if(!TypeIsDefined(res)) res = Resolution.LowVideo; const ent = { Id = theVideoId, Resolution = res, @@ -89,43 +135,20 @@ class TYTD.Downloader { return; } } - - this.LOG($"Adding video: {TYTD.GetVideoId(id)}, original val: {id}"); - switch(res) - { - case Resolution.NoDownload: - { - var id = TYTD.GetVideoId(id); - if(id != null) - this.PutVideoInfoIfNotExists(id); - } - break; - case Resolution.LowVideo: - this.Queue.Push(new TYTD.SDVideoDownload(id)); - break; - case Resolution.VideoOnly: - this.Queue.Push(new TYTD.VOVideoDownload(id)); - break; - - case Resolution.AudioOnly: - this.Queue.Push(new TYTD.AOVideoDownload(id)); - break; - case Resolution.MP4: - this.Queue.Push(new TYTD.TranscodeVideo(id,".mp4")); - break; - case Resolution.MKV: - this.Queue.Push(new TYTD.TranscodeVideo(id,".mkv")); - break; - case Resolution.MP3: - this.Queue.Push(new TYTD.TranscodeAudio(id,".mp3")); - break; - case Resolution.FLAC: - this.Queue.Push(new TYTD.TranscodeAudio(id,".flac")); - break; - case Resolution.DontConvert: - this.Queue.Push(new TYTD.NoConvertVideoDownload(id)); - break; + else { + this.LOG($"Malformed video id {id}"); + return; } + + + this.LOG($"Adding video: https://www.youtube.com/watch?v={theVideoId}"); + //Sqlite.Exec(db,"CREATE TABLE IF NOT EXISTS queue (id INTEGER PRIMARY KEY AUTOINCREMENT, videoId TEXT, resolution TEXT);"); + Mutex.Lock(); + const db = OpenDB(); + Sqlite.Exec(db, $"INSERT INTO queue (videoId, resolution) VALUES ({Sqlite.Escape(theVideoId)},{Sqlite.Escape(res)});"); + Sqlite.Close(db); + Mutex.Unlock(); + } /^ Download a playlist @@ -287,6 +310,18 @@ class TYTD.Downloader { if(views < 1000000000000) return $"{views/1000000000}B views"; return $"{views/1000000000000}T views"; } + + public GetQueueItems(offset, count) + { + this.Mutex.Lock(); + var db = this.OpenDB(); + const vals = Sqlite.Exec(db,$"SELECT * FROM queue ORDER BY id DESC LIMIT {count} OFFSET {offset*count};"); + Sqlite.Close(db); + this.Mutex.Unlock(); + if(TypeIsList(vals)) + return vals; + return []; + } /^ Get videos @@ -295,7 +330,7 @@ class TYTD.Downloader { ^/ public GetVideos(query, offset, count) { - this.Muxex.Lock(); + this.Mutex.Lock(); var db = this.OpenDB(); var q = Sqlite.Escape($"%{query}%"); @@ -330,7 +365,7 @@ class TYTD.Downloader { ^/ public GetPlaylists(query, offset, count) { - this.Muxex.Lock(); + this.Mutex.Lock(); var db = this.OpenDB(); var q = Sqlite.Escape($"%{query}%"); var res = Sqlite.Exec(db, $"SELECT * FROM playlists v WHERE (v.title LIKE {q}) LIMIT {count} OFFSET {offset*count};"); @@ -356,7 +391,7 @@ class TYTD.Downloader { channelTitle="N/A", items = [] }; - this.Muxex.Lock(); + this.Mutex.Lock(); var db = this.OpenDB(); var channelTitle = ""; var title = ""; @@ -420,7 +455,7 @@ class TYTD.Downloader { authorName = "N/A", items = [] }; - this.Muxex.Lock(); + this.Mutex.Lock(); var db = this.OpenDB(); var authorName = ""; @@ -464,7 +499,7 @@ class TYTD.Downloader { ^/ public GetChannels(query, offset, count) { - this.Muxex.Lock(); + this.Mutex.Lock(); var db = this.OpenDB(); var q = Sqlite.Escape($"%{query}%"); var res = Sqlite.Exec(db, $"SELECT * FROM channels v WHERE (v.title LIKE {q}) LIMIT {count} OFFSET {offset*count};"); @@ -524,7 +559,7 @@ class TYTD.Downloader { var id = TYTD.GetVideoId(vid); if(id == null) return null; - this.Muxex.Lock(); + this.Mutex.Lock(); var db = this.OpenDB(); var res = Sqlite.Exec(db, $"SELECT * FROM videos WHERE videoId = {Sqlite.Escape(id)};"); @@ -560,7 +595,7 @@ class TYTD.Downloader { { var id = TYTD.GetPlaylistId(vid); if(id == null) return null; - this.Muxex.Lock(); + this.Mutex.Lock(); var db = this.OpenDB(); var res = Sqlite.Exec(db, $"SELECT * FROM playlists WHERE playlistId = {Sqlite.Escape(id)};"); @@ -582,7 +617,7 @@ class TYTD.Downloader { { var id = TYTD.GetChannelId(vid); if(id == null) return null; - this.Muxex.Lock(); + this.Mutex.Lock(); var db = this.OpenDB(); var res = Sqlite.Exec(db, $"SELECT * FROM channels WHERE channelId = {Sqlite.Escape(id)};"); @@ -599,7 +634,7 @@ class TYTD.Downloader { /^ Get the list of personal list names^/ public GetPersonalLists() { - this.Muxex.Lock(); + this.Mutex.Lock(); var db = this.OpenDB(); var items = []; @@ -621,7 +656,7 @@ class TYTD.Downloader { /^ Set the description of a personal list ^/ public SetPersonalListDescription(name,description) { - this.Muxex.Lock(); + this.Mutex.Lock(); var db = this.OpenDB(); Sqlite.Exec(db, $"UPDATE personal_lists SET description = {Sqlite.Escape(description)} WHERE name = {Sqlite.Escape(name)};"); @@ -631,7 +666,7 @@ class TYTD.Downloader { /^ Get the description of a personal list ^/ public GetPersonalListDescription(name) { - this.Muxex.Lock(); + this.Mutex.Lock(); var db = this.OpenDB(); var res = Sqlite.Exec(db, $"SELECT * FROM personal_lists WHERE name = {Sqlite.Escape(name)};"); @@ -648,7 +683,7 @@ class TYTD.Downloader { } public GetPersonalListTempUrl(name) { - this.Muxex.Lock(); + this.Mutex.Lock(); var db = this.OpenDB(); var items = []; @@ -672,7 +707,7 @@ class TYTD.Downloader { /^ ^/ public GetPersonalListContents(name, offset, count) { - this.Muxex.Lock(); + this.Mutex.Lock(); var db = this.OpenDB(); var items = []; @@ -700,6 +735,23 @@ class TYTD.Downloader { return items; } + + public QueueRemoveItem(id) + { + this.Mutex.Lock(); + const db = OpenDB(); + Sqlite.Exec(db,$"DELETE FROM queue WHERE id = {Sqlite.Escape(id)};"); + Sqlite.Close(db); + this.Mutex.Unlock(); + } + public QueueClear() + { + this.Mutex.Lock(); + const db = OpenDB(); + Sqlite.Exec(db,$"DELETE FROM queue;"); + Sqlite.Close(db); + this.Mutex.Unlock(); + } public AddToPersonalList(name, id) { @@ -822,12 +874,27 @@ class TYTD.Downloader { private DownloaderThreadHandle; - private Queue = new TYTD.Queue(); /^ Get Video Queue count ^/ - public getVideoQueueCount() this.Queue.Count; + public getVideoQueueCount() { + Mutex.Lock(); + const db = OpenDB(); + + const res = Sqlite.Exec(db, "SELECT COUNT(*) FROM queue;"); + + Sqlite.Close(db); + + Mutex.Unlock(); + + const queueCount = res[0].["COUNT(*)"]; + if(TypeIsString(queueCount)) + { + return ParseLong(queueCount); + } + return 0; + } private PlaylistThreadHandle; @@ -851,7 +918,7 @@ class TYTD.Downloader { this.Mutex.Lock(); var db = this.OpenDB(); var _res = Sqlite.Exec(db,$"SELECT * FROM playlists p INNER JOIN playlist_entries e ON p.id = e.playlistId WHERE p.playlistId = {Sqlite.Escape(id)} LIMIT 1;"); - + Sqlite.Close(db); this.Mutex.Unlock(); if(TypeOf(_res) == "List" && _res.Length > 0) { @@ -868,7 +935,7 @@ class TYTD.Downloader { this.Mutex.Lock(); var db = this.OpenDB(); var _res = Sqlite.Exec(db,$"SELECT * FROM videos WHERE channelId = {Sqlite.Escape(id)} LIMIT 1;"); - + Sqlite.Close(db); this.Mutex.Unlock(); if(TypeOf(_res) == "List" && _res.Length > 0) { @@ -1090,7 +1157,12 @@ class TYTD.Downloader { var res = this.GetVideo(videoId); if(res != null) this.Bell.Invoke(this,{ - Video = res + Video = { + VideoId = res.videoId, + ChannelId = res.channelId, + Title = res.title, + Channel = res.author + } }); } if(downloadRes != Resolution.NoDownload) @@ -1134,11 +1206,13 @@ class TYTD.Downloader { while(this.Running) { try { - var res = this.Queue.Pop(); + var res = PopQueue(); + + //Console.WriteLine(res); + if(TypeOf(res) != "Null") { - if(TypeIsDefined(res.TYTD = this)){ res.Progress = (progress)=>{ this.CurrentVideoProgress = progress; @@ -1153,13 +1227,14 @@ class TYTD.Downloader { this.VideoStarted.Invoke(this,{ Video = res.Video }); - res.Start(); this.VideoEnded.Invoke(this,{ Video = res.Video }); + } + } } catch(ex) { try{ @@ -1402,7 +1477,7 @@ class TYTD.Downloader { ^/ public PutVideoInfo(info) { - this.Muxex.Lock(); + this.Mutex.Lock(); var db = this.OpenDB(); var keywords = info.keywords; @@ -1420,7 +1495,7 @@ class TYTD.Downloader { private InitDatabase() { - this.Muxex.Lock(); + this.Mutex.Lock(); var db = this.OpenDB(); Sqlite.Exec(db,"CREATE TABLE IF NOT EXISTS videos (id INTEGER PRIMARY KEY AUTOINCREMENT, videoId TEXT UNIQUE, title TEXT, lengthSeconds INTEGER, keywords TEXT, channelId TEXT, shortDescription TEXT, viewCount INTEGER, author TEXT, addDate INTEGER, tytdTag TEXT);"); Sqlite.Exec(db,"CREATE TABLE IF NOT EXISTS playlists (id INTEGER PRIMARY KEY AUTOINCREMENT, playlistId TEXT UNIQUE,channelId TEXT,channelTitle TEXT, title TEXT);"); @@ -1433,6 +1508,7 @@ class TYTD.Downloader { Sqlite.Exec(db,"CREATE TABLE IF NOT EXISTS users (id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT UNIQUE, password_hash TEXT, password_salt TEXT, flags INTEGER);"); Sqlite.Exec(db,"CREATE TABLE IF NOT EXISTS sessions (id INTEGER PRIMARY KEY AUTOINCREMENT, accountId INTEGER, key TEXT UNIQUE, expires INTEGER);"); Sqlite.Exec(db,"CREATE TABLE IF NOT EXISTS sso (id INTEGER PRIMARY KEY AUTOINCREMENT, service_name TEXT UNIQUE, service_pretty_name TEXT, sso_app_key TEXT UNIQUE, service_auth_post TEXT, service_auth_redirect TEXT);"); + Sqlite.Exec(db,"CREATE TABLE IF NOT EXISTS queue (id INTEGER PRIMARY KEY AUTOINCREMENT, videoId TEXT, resolution TEXT);"); Sqlite.Exec(db,"ALTER TABLE sessions ADD expires INTEGER;"); Sqlite.Exec(db,"DELETE FROM sessions WHERE expires IS NULL;"); var config=Sqlite.Exec(db,"SELECT * FROM plugin_settings WHERE extension = '' AND key = 'settings';"); @@ -1455,7 +1531,7 @@ class TYTD.Downloader { } private _setPluginValue(extension,key,value) { - this.Muxex.Lock(); + this.Mutex.Lock(); var db = this.OpenDB(); Sqlite.Exec(db, $"INSERT OR REPLACE INTO plugin_settings (extension,key,value) VALUES ({Sqlite.Escape(extension)},{Sqlite.Escape(key)},{Sqlite.Escape(value)});"); Sqlite.Close(db); @@ -1701,7 +1777,6 @@ class TYTD.Downloader { ], Body = Net.Http.TextHttpRequestBody(embed("request2.json").ToString().Replace("VIDEO_ID_HERE", id).Replace("VISITOR_DATA",visitor),"application/json") }; - this.RateLimit(); var response = Net.Http.MakeRequest(url,requestData); if(response.StatusCode < 200 || response.StatusCode > 299) { @@ -1734,13 +1809,14 @@ class TYTD.Downloader { if(!TypeIsDictionary(jsonResp.playerResponse.videoDetails)) { + Console.WriteLine("YEY"); throw new VideoDownloadError(id, "videoDetails is missing"); } - if(!TypeIsList(jsonResp.playerResponse.streamingData.adaptiveFormats)) - { - throw new VideoDownloadError(id, "adaptiveFormats is missing"); - } + //if(!TypeIsList(jsonResp.playerResponse.streamingData.adaptiveFormats)) + //{ + // throw new VideoDownloadError(id, "adaptiveFormats is missing"); + //} this.DownloadCaptions(jsonResp); return jsonResp; } @@ -1835,7 +1911,7 @@ class TYTD.Downloader { var first = true; - /*this.Muxex.Lock(); + /*this.Mutex.Lock(); var db = this.OpenDB(); var d = $"INSERT INTO videos (videoId,title,lengthSeconds,keywords,channelId,shortDescription,viewCount,author,addDate,tytdTag) VALUES ({Sqlite.Escape(info.videoId)},{Sqlite.Escape(info.title)},{info.lengthSeconds},{Sqlite.Escape(info.keywords.ToString())},{Sqlite.Escape(info.channelId)},{Sqlite.Escape(info.shortDescription)},{info.viewCount},{Sqlite.Escape(info.author)},{DateTime.NowEpoch},{Sqlite.Escape(this.TYTDTag)});"; Sqlite.Exec(db, d); @@ -1862,7 +1938,7 @@ class TYTD.Downloader { }; */ if(first) { - this.Muxex.Lock(); + this.Mutex.Lock(); var db = this.OpenDB(); if(isPlaylist) { @@ -1898,7 +1974,7 @@ class TYTD.Downloader { ids.Add(vid); if(TypeOf(dbRow) == "Long") { - this.Muxex.Lock(); + this.Mutex.Lock(); var db = this.OpenDB(); Sqlite.Exec(db,$"INSERT INTO playlist_entries (playlistId,videoId) VALUES ({dbRow},{Sqlite.Escape(vid)});"); Sqlite.Close(db); @@ -1919,21 +1995,21 @@ class TYTD.Downloader { private requests = 0; - private rlm=new Muxex(); + private rlm=new Mutex(); private RateLimit() { this.rlm.Lock(); var curRequest = DateTime.NowEpoch; - if((curRequest - this.lastRequest) > 60) + if((curRequest - this.lastRequest) > 10) { this.requests = 0; } this.requests++; - if(this.requests > 5) + if(this.requests >= 1) { - DateTime.Sleep(25000); + DateTime.Sleep(1500); this.requests=0; curRequest = DateTime.NowEpoch; } diff --git a/Tesses.YouTubeDownloader/src/videodownload/audioonlydownload.tcross b/Tesses.YouTubeDownloader/src/videodownload/audioonlydownload.tcross deleted file mode 100644 index 89fe433..0000000 --- a/Tesses.YouTubeDownloader/src/videodownload/audioonlydownload.tcross +++ /dev/null @@ -1,122 +0,0 @@ -class TYTD.AOVideoDownload : IVideoDownload { - private info; - private tytd; - private progress; - private video_stream_url; - - private done=false; - - public AOVideoDownload(id) - { - this.info = { - Title = "", - Channel = "", - VideoId = TYTD.GetVideoId(id), - ChannelId = "" - }; - } - public setTYTD(tytd) - { - this.tytd = tytd; - var id = this.info.VideoId; - var path = /"Streams"/id.Substring(0,4)/id.Substring(4)/"ao.bin"; - if(this.tytd.Storage.FileExists(path)) { - this.done = true; - return null; - } - - var req = this.tytd.ManifestRequest(id).playerResponse; - this.info.Title = req.videoDetails.title; - tytd.LOG($"Downloading: {this.info.Title} with id: {id} Highest Audio"); - Console.WriteLine($"Downloading: {this.info.Title} with id: {id} Highest Audio"); - this.info.Channel = req.videoDetails.author; - this.info.ChannelId = req.videoDetails.channelId; - - this.tytd.PutVideoInfo(req.videoDetails); - - var sampleRate = 0; - var bitrate = 0; - var url = ""; - - each(var item : req.streamingData.adaptiveFormats) - { - if(TypeOf(item.height) != "Long") - { - - item.audioSampleRate = ParseLong(item.audioSampleRate); - if(item.audioSampleRate >= sampleRate && item.bitrate >= bitrate) - { - url = item.url; - sampleRate = item.audioSampleRate; - bitrate = item.bitrate; - } - } - } - - this.video_stream_url = url; - - return tytd; - } - public setProgress(p) - { - this.progress = p; - } - public getVideo() - { - return this.info; - } - public Start() - { - if(this.done) return; - for(var i = 0; i < 5; i++) - { - var req = { - FollowRedirects = true, - RequestHeaders = [ - { - Key = "User-Agent", - - Value = "com.google.android.youtube/19.28.35 (Linux; U; Android 15; GB) gzip" - } - ] - }; - var resp = Net.Http.MakeRequest(this.video_stream_url,req); - - if(resp.StatusCode >= 200 && resp.StatusCode <= 299) - { - var id = this.info.VideoId; - var path = /"Streams"/id.Substring(0,4)/id.Substring(4)/"ao.bin"; - this.tytd.Storage.CreateDirectory(path.GetParent()); - var strm = this.tytd.Storage.OpenFile(path+".part","wb"); - var src = resp.ReadAsStream(); - Helpers.CopyToProgress(src,strm,this.progress,100.0); - strm.Close(); - src.Close(); - this.tytd.Storage.MoveFile(path+".part",path); - break; - }else { - var req = this.tytd.ManifestRequest(this.info.VideoId).playerResponse; - var sampleRate = 0; - var bitrate = 0; - var url = ""; - - each(var item : req.streamingData.adaptiveFormats) - { - if(TypeOf(item.height) != "Long") - { - - item.audioSampleRate = ParseLong(item.audioSampleRate); - if(item.audioSampleRate >= sampleRate && item.bitrate >= bitrate) - { - url = item.url; - sampleRate = item.audioSampleRate; - bitrate = item.bitrate; - } - } - } - - this.video_stream_url = url; - } - } - } -} \ No newline at end of file diff --git a/Tesses.YouTubeDownloader/src/videodownload/noconvertdownload.tcross b/Tesses.YouTubeDownloader/src/videodownload/noconvertdownload.tcross deleted file mode 100644 index 9d600be..0000000 --- a/Tesses.YouTubeDownloader/src/videodownload/noconvertdownload.tcross +++ /dev/null @@ -1,228 +0,0 @@ -class TYTD.NoConvertVideoDownload : IVideoDownload { - private info; - private tytd; - private progress; - private video_stream_url; - private audio_stream_url; - - private done=false; - - - public NoConvertVideoDownload(id) - { - this.info = { - Title = "", - Channel = "", - VideoId = TYTD.GetVideoId(id), - ChannelId = "" - }; - } - public getVideo() - { - return this.info; - } - public setTYTD(tytd) - { - this.tytd = tytd; - var id = this.info.VideoId; - var path = /"Streams"/id.Substring(0,4)/id.Substring(4)/"vo.bin"; - var pathA = /"Streams"/id.Substring(0,4)/id.Substring(4)/"ao.bin"; - if(this.tytd.Storage.FileExists(path) && this.tytd.Storage.FileExists(pathA)) { - this.done = true; - return null; - } - - var req = this.tytd.ManifestRequest(id).playerResponse; - - - this.info.Title = req.videoDetails.title; - tytd.LOG($"Downloading: {this.info.Title} with id: {id} Highest Video/Audio"); - Console.WriteLine($"Downloading: {this.info.Title} with id: {id} Highest Video/Audio"); - this.info.Channel = req.videoDetails.author; - this.info.ChannelId = req.videoDetails.channelId; - - this.tytd.PutVideoInfo(req.videoDetails); - - var width = 0; - var height = 0; - var sampleRate = 0; - var bitrate = 0; - var vurl = ""; - var aurl = ""; - - each(var item : req.streamingData.adaptiveFormats) - { - if(TypeOf(item.height) == "Long") - { - if(item.width >= width && item.height >= height) - { - vurl = item.url; - width = item.width; - height = item.height; - } - } - else { - item.audioSampleRate = ParseLong(item.audioSampleRate); - if(item.audioSampleRate >= sampleRate && item.bitrate >= bitrate) - { - aurl = item.url; - sampleRate = item.audioSampleRate; - bitrate = item.bitrate; - } - } - } - - this.video_stream_url = vurl; - this.audio_stream_url = aurl; - - return tytd; - } - public setProgress(p) - { - this.progress = p; - } - public Start() - { - if(this.done) return; - - var id = this.info.VideoId; - var path = /"Streams"/id.Substring(0,4)/id.Substring(4)/"vo.bin"; - var pathA = /"Streams"/id.Substring(0,4)/id.Substring(4)/"ao.bin"; - - if(!this.tytd.Storage.FileExists(path)) - { - for(var i = 0; i < 5; i++) - { - var req = { - FollowRedirects = true, - RequestHeaders = [ - { - Key = "User-Agent", - - Value = "com.google.android.youtube/19.28.35 (Linux; U; Android 15; GB) gzip" - } - ] - }; - var resp = Net.Http.MakeRequest(this.video_stream_url,req); - - if(resp.StatusCode >= 200 && resp.StatusCode <= 299) - { - this.tytd.Storage.CreateDirectory(path.GetParent()); - var strm = this.tytd.Storage.OpenFile(path+".part","wb"); - var src = resp.ReadAsStream(); - Helpers.CopyToProgress(src,strm,(p)=>{ - this.progress(p/2); - },100.0); - strm.Close(); - src.Close(); - this.tytd.Storage.MoveFile(path+".part",path); - break; - } - else { - var req = this.tytd.ManifestRequest(this.info.VideoId).playerResponse; - var width = 0; - var height = 0; - var sampleRate = 0; - var bitrate = 0; - var vurl = ""; - var aurl = ""; - - each(var item : req.streamingData.adaptiveFormats) - { - if(TypeOf(item.height) == "Long") - { - if(item.width >= width && item.height >= height) - { - vurl = item.url; - width = item.width; - height = item.height; - } - } - else { - item.audioSampleRate = ParseLong(item.audioSampleRate); - if(item.audioSampleRate >= sampleRate && item.bitrate >= bitrate) - { - aurl = item.url; - sampleRate = item.audioSampleRate; - bitrate = item.bitrate; - } - } - } - - this.video_stream_url = vurl; - this.audio_stream_url = aurl; - - } - } - } - if(!this.tytd.Storage.FileExists(pathA)) - { - for(var i = 0; i < 5; i++) - { - var req = { - FollowRedirects = true, - RequestHeaders = [ - { - Key = "User-Agent", - - Value = "com.google.android.youtube/19.28.35 (Linux; U; Android 15; GB) gzip" - } - ] - }; - var resp = Net.Http.MakeRequest(this.audio_stream_url,req); - - if(resp.StatusCode >= 200 && resp.StatusCode <= 299) - { - this.tytd.Storage.CreateDirectory(pathA.GetParent()); - var strm = this.tytd.Storage.OpenFile(pathA+".part","wb"); - var src = resp.ReadAsStream(); - Helpers.CopyToProgress(src,strm,(p)=>{ - this.progress((p/2)+0.5); - },100.0); - strm.Close(); - src.Close(); - this.tytd.Storage.MoveFile(pathA+".part",pathA); - break; - } - else { - var req = this.tytd.ManifestRequest(this.info.VideoId).playerResponse; - var width = 0; - var height = 0; - var sampleRate = 0; - var bitrate = 0; - var vurl = ""; - var aurl = ""; - - each(var item : req.streamingData.adaptiveFormats) - { - if(TypeOf(item.height) == "Long") - { - if(item.width >= width && item.height >= height) - { - vurl = item.url; - width = item.width; - height = item.height; - } - } - else { - item.audioSampleRate = ParseLong(item.audioSampleRate); - if(item.audioSampleRate >= sampleRate && item.bitrate >= bitrate) - { - aurl = item.url; - sampleRate = item.audioSampleRate; - bitrate = item.bitrate; - } - } - } - - this.video_stream_url = vurl; - this.audio_stream_url = aurl; - - } - } - } - this.progress(1.0); - - return; - } -} \ No newline at end of file diff --git a/Tesses.YouTubeDownloader/src/videodownload/sdvideodownload.tcross b/Tesses.YouTubeDownloader/src/videodownload/sdvideodownload.tcross index b5a6e6a..db18de3 100644 --- a/Tesses.YouTubeDownloader/src/videodownload/sdvideodownload.tcross +++ b/Tesses.YouTubeDownloader/src/videodownload/sdvideodownload.tcross @@ -20,8 +20,16 @@ class TYTD.SDVideoDownload : IVideoDownload { this.tytd = tytd; var id = this.info.VideoId; var path = /"Streams"/id.Substring(0,4)/id.Substring(4)/"ytmux.mp4"; - if(this.tytd.Storage.FileExists(path)) { + if(this.tytd.Storage.FileExists(path)) { this.done = true; + const vinfo = this.tytd.GetVideo(id); + this.info = { + Title = vinfo.title, + VideoId = vinfo.videoId, + ChannelId = vinfo.channelId, + Channel = vinfo.author + }; + return null; } diff --git a/Tesses.YouTubeDownloader/src/videodownload/transcodeamv.tcross b/Tesses.YouTubeDownloader/src/videodownload/transcodeamv.tcross new file mode 100644 index 0000000..f827f2c --- /dev/null +++ b/Tesses.YouTubeDownloader/src/videodownload/transcodeamv.tcross @@ -0,0 +1,84 @@ +class TYTD.TranscodeAMV : IVideoDownload { + private id; + private ncv; + private tytd; + private done; + + public TranscodeAMV(id) + { + this.id = id; + this.ncv = new TYTD.SDVideoDownload(id); + } + + public setTYTD(tytd) + { + this.tytd = tytd; + this.ncv.TYTD = tytd; + + if(TypeIsDefined(tytd)) + { + var dir = this.tytd.DatabaseDirectory / "Streams"/this.id.Substring(0,4)/this.id.Substring(4); + var ao = dir / "ytmux.mp4"; + var out = dir /"conv.amv"; + + if(FS.Local.FileExists(out)) return null; + return tytd; + } + + return null; + } + + public setProgress(p) + { + return this.ncv.Progress = p; + } + + public getVideo() + { + return this.ncv.Video; + } + + public Start() + { + var id = this.id; + this.ncv.Start(); + + var p = new Process(); + p.FileName = Env.GetRealExecutablePath("ffmpeg").ToString(); + + var dir = this.tytd.DatabaseDirectory / "Streams"/this.id.Substring(0,4)/this.id.Substring(4); + var ao = dir / "ytmux.mp4"; + var out = dir / $"conv.amv"; + if(FS.Local.FileExists(out) || !FS.Local.FileExists(ao)) return; + var args=["-y","-i",ao.ToString()]; + //-c:v amv -c:a adpcm_ima_amv -pix_fmt yuvj420p -vstrict -1 -s 160x120 -ac 1 -ar 22050 -r 16 -block_size 882 + + + args.Add("-c:v"); + args.Add("amv"); + args.Add("-c:a"); + args.Add("adpcm_ima_amv"); + args.Add("-pix_fmt"); + args.Add("yuvj420p"); + args.Add("-vstrict"); + args.Add("-1"); + args.Add("-s"); + args.Add("160x120"); + args.Add("-ac"); + args.Add("1"); + args.Add("-ar"); + args.Add("22050"); + args.Add("-r"); + args.Add("15"); + args.Add("-block_size"); + args.Add("1470"); + args.Add("-preset"); + args.Add("ultrafast"); + args.Add(out.ToString()); + + + p.Arguments = args; + if(p.Start()) + p.Join(); + } +} \ No newline at end of file diff --git a/Tesses.YouTubeDownloader/src/videodownload/transcodeaudio.tcross b/Tesses.YouTubeDownloader/src/videodownload/transcodeaudio.tcross index 5fc4aa0..94f2b0e 100644 --- a/Tesses.YouTubeDownloader/src/videodownload/transcodeaudio.tcross +++ b/Tesses.YouTubeDownloader/src/videodownload/transcodeaudio.tcross @@ -9,13 +9,24 @@ class TYTD.TranscodeAudio : IVideoDownload { { this.id = id; this.ext = ext; - this.ncv = new TYTD.AOVideoDownload(id); + this.ncv = new TYTD.SDVideoDownload(id); } public setTYTD(tytd) { this.tytd = tytd; - return this.ncv.TYTD = tytd; + this.ncv.TYTD = tytd; + + if(TypeIsDefined(tytd)) + { + var dir = this.tytd.DatabaseDirectory / "Streams"/this.id.Substring(0,4)/this.id.Substring(4); + var out = dir /"conv.{this.ext}"; + + if(FS.Local.FileExists(out)) return null; + return tytd; + } + + return null; } public setProgress(p) @@ -31,15 +42,15 @@ class TYTD.TranscodeAudio : IVideoDownload { public Start() { var id = this.id; - this.mcv.Start(); + this.ncv.Start(); var p = new Process(); p.FileName = Env.GetRealExecutablePath("ffmpeg").ToString(); var dir = this.tytd.DatabaseDirectory / "Streams"/this.id.Substring(0,4)/this.id.Substring(4); - var ao = dir / "ao.bin"; + var ao = dir / "ytmux.mp4"; var out = dir / $"conv{this.ext}"; - if(FS.Local.FileExists(out)) return; + if(FS.Local.FileExists(out) && !FS.Local.FileExists(ao)) return; var args=["-y","-i",ao.ToString()]; diff --git a/Tesses.YouTubeDownloader/src/videodownload/transcodevideo.tcross b/Tesses.YouTubeDownloader/src/videodownload/transcodevideo.tcross deleted file mode 100644 index f44073c..0000000 --- a/Tesses.YouTubeDownloader/src/videodownload/transcodevideo.tcross +++ /dev/null @@ -1,63 +0,0 @@ -class TYTD.TranscodeVideo : IVideoDownload { - private id; - private ncv; - private tytd; - private ext; - - public TranscodeVideo(id,ext) - { - this.id = id; - this.ext = ext; - this.ncv = new TYTD.NoConvertVideoDownload(id); - } - - public setTYTD(tytd) - { - this.tytd = tytd; - return this.ncv.TYTD = tytd; - } - - public setProgress(p) - { - return this.ncv.Progress = p; - } - - public getVideo() - { - return this.ncv.Video; - } - - public Start() - { - var id = this.id; - this.mcv.Start(); - - var p = new Process(); - p.FileName = Env.GetRealExecutablePath("ffmpeg").ToString(); - - var dir = this.tytd.DatabaseDirectory / "Streams"/this.id.Substring(0,4)/this.id.Substring(4); - var vo = dir / "vo.bin"; - var ao = dir / "ao.bin"; - var out = dir / $"conv{this.ext}"; - if(FS.Local.FileExists(out)) return; - var args=["-y","-i",vo.ToString(),"-i",ao.ToString()]; - - if(this.ext == ".mkv") - { - args.Add("-c"); - args.Add("copy"); - } - args.Add("-map"); - args.Add("0:v"); - args.Add("-map"); - args.Add("1:a"); - args.Add("-preset"); - args.Add("ultrafast"); - args.Add(out.ToString()); - - - p.Arguments = args; - if(p.Start()) - p.Join(); - } -} \ No newline at end of file diff --git a/Tesses.YouTubeDownloader/src/videodownload/videoonlydownload.tcross b/Tesses.YouTubeDownloader/src/videodownload/videoonlydownload.tcross deleted file mode 100644 index 2a48053..0000000 --- a/Tesses.YouTubeDownloader/src/videodownload/videoonlydownload.tcross +++ /dev/null @@ -1,115 +0,0 @@ -class TYTD.VOVideoDownload : IVideoDownload { - private info; - private tytd; - private progress; - private video_stream_url; - - private done=false; - - public VOVideoDownload(id) - { - this.info = { - Title = "", - Channel = "", - VideoId = TYTD.GetVideoId(id), - ChannelId = "" - }; - } - public setTYTD(tytd) - { - this.tytd = tytd; - var id = this.info.VideoId; - var path = /"Streams"/id.Substring(0,4)/id.Substring(4)/"vo.bin"; - if(this.tytd.Storage.FileExists(path)) { - this.done = true; - return null; - } - - var req = this.tytd.ManifestRequest(id).playerResponse; - this.info.Title = req.videoDetails.title; - this.info.Channel = req.videoDetails.author; - this.info.ChannelId = req.videoDetails.channelId; - tytd.LOG($"Downloading: {this.info.Title} with id: {id} Highest Video"); - Console.WriteLine($"Downloading: {this.info.Title} with id: {id} Highest Video"); - - this.tytd.PutVideoInfo(req.videoDetails); - - var width = 0; - var height = 0; - var url = ""; - - each(var item : req.streamingData.adaptiveFormats) - { - if(TypeOf(item.height) == "Long") - { - if(item.width >= width && item.height >= height) - { - url = item.url; - width = item.width; - height = item.height; - } - } - } - - this.video_stream_url = url; - - return tytd; - } - public setProgress(p) - { - this.progress = p; - } - public getVideo() - { - return this.info; - } - public Start() - { - if(this.done) return; - var req = { - FollowRedirects = true, - RequestHeaders = [ - { - Key = "User-Agent", - - Value = "com.google.android.youtube/19.28.35 (Linux; U; Android 15; GB) gzip" - } - ] - }; - var resp = Net.Http.MakeRequest(this.video_stream_url,req); - - if(resp.StatusCode >= 200 && resp.StatusCode <= 299) - { - var id = this.info.VideoId; - var path = /"Streams"/id.Substring(0,4)/id.Substring(4)/"vo.bin"; - this.tytd.Storage.CreateDirectory(path.GetParent()); - var strm = this.tytd.Storage.OpenFile(path+".part","wb"); - var src = resp.ReadAsStream(); - Helpers.CopyToProgress(src,strm,this.progress,100.0); - strm.Close(); - src.Close(); - this.tytd.Storage.MoveFile(path+".part",path); - } - else { - var req = this.tytd.ManifestRequest(this.info.VideoId).playerResponse; - var width = 0; - var height = 0; - var url = ""; - - each(var item : req.streamingData.adaptiveFormats) - { - if(TypeOf(item.height) == "Long") - { - if(item.width >= width && item.height >= height) - { - url = item.url; - width = item.width; - height = item.height; - } - } - } - - this.video_stream_url = url; - } - } -} \ No newline at end of file diff --git a/build-for-system-pkg.sh b/build-for-system-pkg.sh new file mode 100644 index 0000000..fe5c479 --- /dev/null +++ b/build-for-system-pkg.sh @@ -0,0 +1,18 @@ +cd Tesses.YouTubeDownloader.Server +crosslang build +cd bin +cp Tesses.YouTubeDownloader.Server-*.crvm tytd2025.crvm +cd ../.. +mkdir -p packages-tmp +nfpm package -t packages-tmp -p archlinux +curl --user tesses50:$PACKAGE_AND_BREW \ + --upload-file packages-tmp/*.pkg.tar.zst \ + https://git.tesses.org/api/packages/tesses50/arch/core +nfpm package -t packages-tmp -p deb +curl --user tesses50:$PACKAGE_AND_BREW \ + --upload-file packages-tmp/*.deb \ + https://git.tesses.org/api/packages/tesses50/debian/pool/universal/main/upload +nfpm package -t packages-tmp -p rpm +curl --user tesses50:$PACKAGE_AND_BREW \ + --upload-file packages-tmp/*.rpm \ + https://git.tesses.org/api/packages/tesses50/rpm/upload \ No newline at end of file diff --git a/changeme.md b/changeme.md new file mode 100644 index 0000000..a17d787 --- /dev/null +++ b/changeme.md @@ -0,0 +1,4 @@ +# Changelog + +## 1.0.0 +Make it a little faster, queue is in sqlite now, fix db bug and more diff --git a/docker-compose.yml b/docker-compose.yml index e3134c6..8d3beb6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: tytd: - image: onedev.site.tesses.net/tytd2025/tytd2025:latest + image: git.tesses.org/tesses50/tytd2025:latest volumes: - tytd-data:/data ports: diff --git a/nfpm.yaml b/nfpm.yaml new file mode 100644 index 0000000..3bc953c --- /dev/null +++ b/nfpm.yaml @@ -0,0 +1,43 @@ +# This is an example nfpm configuration file. +# Make sure to check the documentation at https://nfpm.goreleaser.com +# +# The lines below are called `modelines`. See `:help modeline` +# Feel free to remove those if you don't want/need to use them. +# yaml-language-server: $schema=https://nfpm.goreleaser.com/schema.json +# vim: set ts=2 sw=2 tw=0 fo=cnqoj + +name: "tytd2025" +arch: "any" +platform: "linux" +version: "1.0.0" +section: "default" +priority: "extra" + +depends: + - crosslang +recommends: + - ffmpeg +maintainer: "Mike Nolan " +description: Tesses YouTube Downloader +vendor: "Tesses" +homepage: "https://git.tesses.org/tesses50/tytd2025.git" +license: "GPLv3" +contents: + - src: "Tesses.YouTubeDownloader.Server/bin/tytd2025.crvm" + dst: "/opt/tytd2025/tytd2025.crvm" + - src: "Tesses.YouTubeDownloader.Server/bin/Tesses.YouTubeDownloader-*.crvm" + dst: "/opt/tytd2025/" + - src: "Tesses.YouTubeDownloader.Server/bin/Tesses.CrossLang.BuildEssentials-*.crvm" + dst: "/opt/tytd2025/" + - src: package/systemd/ + dst: /etc/systemd/system/ + type: tree + - src: package/usrbin/ + dst: /usr/bin/ + type: tree + file_info: + mode: 0755 + +scripts: + postinstall: ./package/postinstall.sh + preremove: ./package/preremove.sh diff --git a/package/postinstall.sh b/package/postinstall.sh new file mode 100644 index 0000000..949f72c --- /dev/null +++ b/package/postinstall.sh @@ -0,0 +1,5 @@ +useradd --system --create-home --shell /usr/sbin/nologin tytd2025 || true + +if command -v deb-systemd-helper >/dev/null 2>&1; then + deb-systemd-helper enable tytd2025 +fi \ No newline at end of file diff --git a/package/preremove.sh b/package/preremove.sh new file mode 100644 index 0000000..cab4ba4 --- /dev/null +++ b/package/preremove.sh @@ -0,0 +1,3 @@ +if command -v deb-systemd-helper >/dev/null 2>&1; then + deb-systemd-helper disable tytd2025 +fi \ No newline at end of file diff --git a/package/systemd/tytd2025.service b/package/systemd/tytd2025.service new file mode 100644 index 0000000..7cb9e63 --- /dev/null +++ b/package/systemd/tytd2025.service @@ -0,0 +1,14 @@ +[Unit] +Description=Tesses YouTube Downloader +Documentation=https://git.tesses.org/tesses50/tytd2025 + + +[Service] +Type=simple +ExecStart=/usr/bin/tytd2025 +Restart=on-failure +User=tytd2025 +StandardOutput=journal + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/package/usrbin/tytd2025 b/package/usrbin/tytd2025 new file mode 100755 index 0000000..082f0da --- /dev/null +++ b/package/usrbin/tytd2025 @@ -0,0 +1,2 @@ +#!/usr/bin/env sh +exec crosslang vm "/opt/tytd2025/tytd2025.crvm" --port=3255 \ No newline at end of file