pisto/ncdownloader
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added options for tracking youtube-dl downloads;bugs fixing

Browse Source
This commit is contained in:
huangjx
2021-09-14 22:58:36 +08:00
parent ced82caf0a
commit 2e834d4f9c
23 changed files with 649 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
lib/Tools/Aria2.php
View File

@@ -162,8 +162,11 @@ class Aria2
{
$this->filterResponse = false;
$resp = $this->tellStopped($range);
if (!isset($resp['result'])) {
return [];
}
$result = $this->sortDownloadsResult($resp['result'], ['complete', 'removed']);
$this->filterResponse = true;;
$this->filterResponse = true;
return $result;
}
public function getCounters()
@@ -268,6 +271,21 @@ class Aria2
$this->start();
}
public function download(String $url)
{
$resp = $this->addUri([$url]);
if (isset($resp['error'])) {
return $resp;
}
if (isset($resp['result']) && is_string($gid = $resp['result'])) {
return $gid;
}
return false;
}
public function getDefaults()
{
return [

23
lib/Tools/DBConn.php
View File

@@ -18,7 +18,7 @@ class DBConn
//$this->conn = $this->connAdapter->getInner();
}
public function create($insert)
public function insert($insert)
{
$inserted = (bool) $this->conn->insertIfNotExist('*PREFIX*' . $this->table, $insert, [
'gid',
@@ -46,6 +46,19 @@ class DBConn
return $queryBuilder->fetchAll();
}
public function getYoutubeByUid($uid)
{
$queryBuilder = $this->queryBuilder
->select('*')
->from($this->table)
->where('uid = :uid')
->where('type = :type')
->setParameter('uid', $uid)
->setParameter('type', 2)
->execute();
return $queryBuilder->fetchAll();
}
public function getByGid($gid)
{
$queryBuilder = $this->queryBuilder
@@ -57,9 +70,9 @@ class DBConn
return $queryBuilder->fetch();
}
public function save(array $keys, $values = array())
public function save(array $keys, $values = array(),$conditions = array())
{
return $this->conn->setValues($this->table, $keys, $values);
return $this->conn->setValues($this->table, $keys, $values,$conditions);
}
public function deleteByGid($gid)
@@ -74,14 +87,14 @@ class DBConn
}
public function execute($sql, $values)
{
return $this->conn->executeStatement($sql, $values);
return $this->conn->executeUpdate($sql, $values);
// for some reason this doesn't work
$query = $this->queryBuilder;
$query->update('ncdownloader_info')
->set("data", $query->createNamedParameter($value))
->where($query->expr()->eq('gid', $query->createNamedParameter($gid)));
// ->setParameter('gid', $gid);
// ->setParameter('gid', $gid);
// return $query->execute();
//return $query->getSQL();
return $this->queryBuilder->getSQL();

42
lib/Tools/File.php
View File

@@ -1,42 +0,0 @@
<?php
namespace OCA\NCDownloader\Tools;
use OCA\NCDownloader\Tools\Helper;
use OCA\NCDownloader\Tools\Settings;
use OC\Files\Utils\Scanner;
use \OCP\EventDispatcher\IEventDispatcher;
class File
{
public static function syncFolder($dir = null)
{
$user = \OC::$server->getUserSession()->getUser()->getUID();
if (!isset($dir)) {
$settings = new Settings($user);
$downloadDir = $settings->get('ncd_downloader_dir') ?? "/Downloads";
$rootFolder = Helper::getUserFolder($user);
$path = $rootFolder . "/" . ltrim($downloadDir, '/\\');
} else {
$path = $dir;
}
$realDir =\OC::$server->getSystemConfig()->getValue('datadirectory') . "/" . $path;
if (!(Helper::folderUpdated($realDir))) {
return ['message' => "no change"];
}
$logger = \OC::$server->getLogger();
$scanner = new Scanner($user, \OC::$server->getDatabaseConnection(), \OC::$server->query(IEventDispatcher::class), $logger);
try {
$scanner->scan($path);
// Helper::debug($logger->getLogPath());
//$logger->warning($logger->getLogPath(),['app' =>'Ncdownloader']);
} catch (ForbiddenException $e) {
$logger->warning("Make sure you're running the scan command only as the user the web server runs as");
} catch (\Exception $e) {
$logger->warning("Exception during scan: " . $e->getMessage() . $e->getTraceAsString());
}
return ['message' => "changed"];
}
}

2
lib/Tools/Helper.php
View File

@@ -8,7 +8,7 @@ use OC\Files\Filesystem;
class Helper
{
public const DOWNLOADTYPE = ['ARIA2' => 1, 'YOUTUBE-DL' => 2, 'OTHERS' => 3];
public const STATUS = ['ACTIVE' => 1, 'ERROR' => 2, 'COMPLETE' => 3];
public const STATUS = ['ACTIVE' => 1, 'PAUSED' => 2, 'COMPLETE' => 3, 'ERROR' => 4];
public static function isUrl($URL)
{
$URLPattern = '%^(?:(?:https?|ftp)://)(?:\S+(?::\S*)?@|\d{1,3}(?:\.\d{1,3}){3}|(?:(?:[a-z\d\x{00a1}-\x{ffff}'

66
lib/Tools/Youtube.php
View File

@@ -2,6 +2,7 @@
namespace OCA\NCDownloader\Tools;
use OCA\NCDownloader\Tools\Helper;
use OCA\NCDownloader\Tools\YoutubeHelper;
use Symfony\Component\Process\Exception\ProcessFailedException;
use Symfony\Component\Process\Process;
@@ -12,11 +13,22 @@ class Youtube
private $audioFormat, $videoFormat = 'mp4';
private $options = [];
private $downloadDir;
private $timeout = 60 * 60 * 15;
private $outTpl = "/%(id)s-%(title)s.%(ext)s";
private $defaultDir = "/tmp/downloads";
public function __construct($config)
{
$config += ['downloadDir' => '/tmp/downloads'];
$this->bin = Helper::findBinaryPath('youtube-dl');
$this->init();
$this->setDownloadDir($config['downloadDir']);
$this->helper = YoutubeHelper::create();
}
public function init()
{
$this->addOption("--no-mtime");
}
public function GetUrlOnly()
@@ -46,38 +58,80 @@ class Youtube
array_unshift($this->options, $option);
}
public function download($url)
public function downloadSync($url)
{
$this->downloadDir = $this->downloadDir ?? "/tmp/downloads";
$this->prependOption($this->downloadDir . "/%(id)s-%(title)s.%(ext)s");
$this->downloadDir = $this->downloadDir ?? $this->defaultDir;
$this->prependOption($this->downloadDir . $this->outTpl);
$this->prependOption("-o");
$this->setUrl($url);
$this->prependOption($this->bin);
// $this->buildCMD();
$process = new Process($this->options);
//the maximum time required to download the file
$process->setTimeout(60*60*15);
$process->setTimeout($this->timeout);
try {
$process->mustRun();
$output = $process->getOutput();
} catch (ProcessFailedException $exception) {
$output = $exception->getMessage();
}
return $output;
}
public function download($url)
{
$this->downloadDir = $this->downloadDir ?? $this->defaultDir;
$this->prependOption($this->downloadDir . $this->outTpl);
$this->prependOption("-o");
$this->setUrl($url);
$this->prependOption($this->bin);
$process = new Process($this->options);
$process->setTimeout($this->timeout);
$process->run(function ($type, $buffer) use ($url) {
if (Process::ERR === $type) {
$this->onError($buffer);
} else {
$this->onOutput($buffer, $url);
}
});
if ($process->isSuccessful()) {
$this->helper->updateStatus(Helper::STATUS['COMPLETE']);
return ['message' => $this->helper->file ?? $process->getErrorOutput()];
}
return $process->getErrorOutput();
}
public function getFilePath($output)
{
$rules = '#\[download\]\s+Destination:\s+(?<filename>.*\.(?<ext>(mp4|mp3|aac)))$#i';
preg_match($rules, $output, $matches);
return $matches['filename'] ?? null;
}
private function onError($buffer)
{
$this->helper->log($buffer);
}
public function onOutput($buffer, $url)
{
$this->helper->run($buffer, $url);
}
public function getDownloadUrl($url)
{
$this->setUrl($url);
$this->GetUrlOnly();
//$process = new Process($this->options);
$this->buildCMD();
exec($this->cmd, $output, $returnCode);
if (count($output) === 1) {
return ['url' => reset($output)];
}
list($url, $filename) = $output;
return ['url' => $url, 'filename' => Helper::cleanString($filename)];
$filename = Helper::cleanString($filename);
return ['url' => $url, 'filename' => Helper::clipFilename($filename)];
}
public function setUrl($url)

92
lib/Tools/YoutubeHelper.php Normal file
View File

@@ -0,0 +1,92 @@
<?php
namespace OCA\NCDownloader\Tools;
use OCA\NCDownloader\Tools\DBConn;
use OCA\NCDownloader\Tools\Helper;
class YoutubeHelper
{
public const PROGRESS_PATTERN = '#\[download\]\s+' .
'(?<percentage>\d+(?:\.\d+)?%)' . //progress
'\s+of\s+[~]?' .
'(?<size>\d+(?:\.\d+)?(?:K|M|G)iB)' . //file size
'(?:\s+at\s+' .
'(?<speed>(\d+(?:\.\d+)?(?:K|M|G)iB/s)|Unknown speed))' . //speed
'(?:\s+ETA\s+(?<eta>([\d:]{2,8}|Unknown ETA)))?' . //estimated download time
'(\s+in\s+(?<totalTime>[\d:]{2,8}))?#i';
public $file = null;
public $filesize = null;
public function __construct()
{
$this->dbconn = new DBConn();
$this->tablename = $this->dbconn->queryBuilder->getTableName("ncdownloader_info");
$this->user = \OC::$server->getUserSession()->getUser()->getUID();
}
public static function create()
{
return new static();
}
public function getFilePath($output)
{
$rules = '#\[download\]\s+Destination:\s+(?<filename>.*\.(?<ext>(mp4|mp3|aac)))$#i';
preg_match($rules, $output, $matches);
return $matches['filename'] ?? null;
}
public function log($message)
{
Helper::debug($message);
}
public function updateStatus($status = null)
{
if (isset($status)) {
$this->status = trim($status);
}
$sql = sprintf("UPDATE %s set status = ? WHERE gid = ?", $this->tablename);
$this->dbconn->execute($sql, [$this->status, $this->gid]);
}
public function run($buffer, $url)
{
$this->gid = Helper::generateGID($url);
$file = $this->getFilePath($buffer);
if ($file) {
$data = [
'uid' => $this->user,
'gid' => $this->gid,
'type' => Helper::DOWNLOADTYPE['YOUTUBE-DL'],
'filename' => basename($file),
'status' => Helper::STATUS['ACTIVE'],
'timestamp' => time(),
'data' => serialize(['link' => $url]),
];
//save the filename as this runs only once
$this->file = $file;
$this->dbconn->insert($data);
//$this->dbconn->save($data,[],['gid' => $this->gid]);
}
if (preg_match_all(self::PROGRESS_PATTERN, $buffer, $matches, PREG_SET_ORDER) !== false) {
if (count($matches) > 0) {
$match = reset($matches);
//save the filesize
if (!isset($this->filesize) && isset($match['size'])) {
$this->filesize = $match['size'];
}
$size = $match['size'];
$percentage = $match['percentage'];
$speed = $match['speed'] . "|" . $match['eta'];
$sql = sprintf("UPDATE %s set filesize = ?,speed = ?,progress = ? WHERE gid = ?", $this->tablename);
$this->dbconn->execute($sql, [$this->filesize, $speed, $percentage, $this->gid]);
/* $data = [
'filesize' => $size,
'speed' => $speed,
'progress' => $percentage,
'gid' => $this->gid,
];
$this->dbconn->save([], $data, ['gid' => $this->gid]);*/
}
}
}
}

93
lib/Tools/folderScan.php Normal file
View File

@@ -0,0 +1,93 @@
<?php
namespace OCA\NCDownloader\Tools;
use OCA\NCDownloader\Tools\Helper;
use OCA\NCDownloader\Tools\Settings;
use OC\Files\Utils\Scanner;
use \OCP\EventDispatcher\IEventDispatcher;
class folderScan
{
private $user;
private $path;
public function __construct($path = null, $user = null)
{
$this->user = $user ?? \OC::$server->getUserSession()->getUser()->getUID();
$this->path = $path ?? $this->getDefaultPath();
$this->realDir = \OC::$server->getSystemConfig()->getValue('datadirectory') . "/" . $this->path;
}
public function getDefaultPath()
{
$settings = new Settings($this->user);
$rootFolder = Helper::getUserFolder($this->user);
$downloadDir = $settings->get('ncd_downloader_dir') ?? "/Downloads";
return $rootFolder . "/" . ltrim($downloadDir, '/\\');
}
public static function create($path = null, $user = null)
{
return new static($path, $user);
}
public function setUser($user)
{
$this->user = $user;
return $this;
}
public function setPath($path)
{
$this->path = $path;
return $this;
}
private function update()
{
if (!(self::folderUpdated($this->realDir))) {
return ['message' => "no change"];
}
$this->scan();
return ['message' => "changed"];
}
//force update
public function scan()
{
$this->logger = \OC::$server->getLogger();
$this->scanner = new Scanner($this->user, \OC::$server->getDatabaseConnection(), \OC::$server->query(IEventDispatcher::class), $this->logger);
try {
$this->scanner->scan($this->path);
return ['status' => 'OK', 'path' => $this->path];
} catch (ForbiddenException $e) {
$this->logger->warning("Make sure you're running the scan command only as the user the web server runs as");
} catch (\Exception $e) {
$this->logger->warning("Exception during scan: " . $e->getMessage() . $e->getTraceAsString());
}
return ['status' => $e->getMessage(), 'path' => $this->path];
}
public static function folderUpdated($dir)
{
if (!file_exists($dir)) {
return false;
}
$checkFile = $dir . "/.lastmodified";
if (!file_exists($checkFile)) {
$time = \filemtime($dir);
file_put_contents($checkFile, $time);
return false;
}
$lastModified = (int) file_get_contents($checkFile);
$time = \filemtime($dir);
if ($time > $lastModified) {
file_put_contents($checkFile, $time);
return true;
}
return false;
}
//update only folder is modified
public static function sync($path = null, $user = null)
{
return self::create($path, $user)->update();
}
}