Fix for 9.x upgrade + better handling of Anistats rate limiting

6 files changed, 48 insertions, 14 deletions

diff --git a/app/Anime.php b/app/Anime.php
index 4999c1b..ab5a3fe 100644
--- a/app/Anime.php
+++ b/app/Anime.php
@@ -5,6 +5,7 @@
 
 namespace App;
 use Illuminate\Database\Eloquent\Model;
+use Illuminate\Support\Facades\Log;
 
 use Jikan\MyAnimeList\MalClient;
 
@@ -34,7 +35,7 @@ class Anime extends Model {
 	public function __construct() {
 	}
 
-	public function fill( $id, $skip_if_unpopular = true ) {
+	public function fillStats( $id, $skip_if_unpopular = true ) {
 		$this->mal_id = $id;
 
 		$jikan = new Malclient;
@@ -143,7 +144,7 @@ class Anime extends Model {
 
 			$data = json_decode( $response->getBody() )->data->Media;
 		} catch (\Exception $e) {
-			echo "Problem with Guzzle connecting to Anilist on anime: (" . $this->mal_id . ")\n";
+			echo "getDataFromAnilist: Problem with Guzzle connecting to Anilist on anime: (" . $this->mal_id . ")\n";
 			return "";
 		}
 		return $data;
@@ -178,10 +179,22 @@ class Anime extends Model {
 			]);
 
 			$data = json_decode( $response->getBody() )->data->Media;
+
+			$rate_limit = $response->getHeader("X-RateLimit-Remaining")[0];
+			echo "Try Block: Rate Limit: " . $rate_limit . "\n";
+			Log::info("getEnhancementFromAnilist: Try Block: Rate Limit: " . $rate_limit . "\n");
+			if ( $rate_limit <= 10 ) {
+				echo " sleep 5 seconds\n\n";
+				sleep(5);
+			}
 		} catch (\Exception $e) {
-			echo "Problem with Guzzle connecting to Anilist on anime: (" . $this->mal_id . ")\n";
+			echo "getEnhancementFromAnilist: Problem with Guzzle connecting to Anilist on anime: (" . $this->mal_id . ")\n";
+			echo $e->getMessage();
+			echo "\n";
 			return "";
 		}
+		echo "got enhancement for anime " . $this->mal_id . "\n\n";
+		Log::info("got enhancement for anime " . $this->mal_id . "\n");
 		return $data;
 
 	}
diff --git a/app/AnimeStats.php b/app/AnimeStats.php
index 9b9b850..c235a00 100644
--- a/app/AnimeStats.php
+++ b/app/AnimeStats.php
@@ -76,7 +76,7 @@ class AnimeStats extends Model {
 	public function __construct() {
 	}
 
-	public function fill( $id, $season_year = 0, $season_name = "" ) {
+	public function fillStats( $id, $season_year = 0, $season_name = "" ) {
 		$this->mal_id = $id;
 
 		$jikan = new Malclient;
diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php
index 01ae9ac..fc949d4 100644
--- a/app/Console/Kernel.php
+++ b/app/Console/Kernel.php
@@ -36,10 +36,10 @@ class Kernel extends ConsoleKernel
 	$schedule->call( function(){
 		$background = new Background();
 		$background->saveAnimeStats();
-	})->everyThirtyMinutes()->name('saveStats')->withoutOverlapping();
+	})->everyThirtyMinutes()->name('saveStats2')->withoutOverlapping();
 	#})->everyThirtyMinutes()->name('saveStats5')->withoutOverlapping();
 	#})->everyMinute()->name('saveStats3')->withoutOverlapping();
-	#})->everyMinute()->name('saveStats5')->withoutOverlapping();
+	#})->everyMinute()->name('saveStats5');
 
 	$schedule->call( function(){
 		$background = new Background();
@@ -75,6 +75,7 @@ class Kernel extends ConsoleKernel
 		echo "addSeasonToAllAnime()\n";
 		$background->addSeasonToAllAnime();
 	})->weeklyOn(1, '15:30')->name('saveTopAnime')->withoutOverlapping();
+	#})->everyMinute();
 
 	$schedule->call( function(){
 		$background = new Background();
@@ -88,11 +89,12 @@ class Kernel extends ConsoleKernel
 		$background = new Background();
 		#$background->checkIfIsAiring();
 		#$background->saveSeason();
-		#$background->saveEnhancementForAll();
+		$background->saveEnhancementForAll();
 		#$background->addSeasonToAllAnime();
 		#$anime = Anime::where('mal_id', 40591)->get()->first();
 		#$background->saveAiring( $anime );
-	})->everyMinute()->name("enhance6")->withoutOverlapping();
+	})->everyMinute()->name("enhance3")->withoutOverlapping();
+	#})->everyMinute()->name("enhance6");
 	#})->twiceDaily(9, 23);
 	*/
 
diff --git a/app/Http/Middleware/TrustProxies.php b/app/Http/Middleware/TrustProxies.php
index 3625f37..f8baead 100644
--- a/app/Http/Middleware/TrustProxies.php
+++ b/app/Http/Middleware/TrustProxies.php
@@ -19,5 +19,10 @@ class TrustProxies extends Middleware
      *
      * @var int
      */
-    protected $headers = Request::HEADER_X_FORWARDED_ALL;
+    protected $headers =
+	    Request::HEADER_X_FORWARDED_FOR |
+	    Request::HEADER_X_FORWARDED_HOST |
+	    Request::HEADER_X_FORWARDED_PORT |
+	    Request::HEADER_X_FORWARDED_PROTO |
+	    Request::HEADER_X_FORWARDED_AWS_ELB;
 }
diff --git a/app/Libraries/Background.php b/app/Libraries/Background.php
index 91d9d75..3f8ea5c 100644
--- a/app/Libraries/Background.php
+++ b/app/Libraries/Background.php
@@ -156,7 +156,7 @@ class Background {
 
 			$animeStats = new AnimeStats();
 
-			if ( $animeStats->fill($entry->mal_id) ) {
+			if ( $animeStats->fillStats($entry->mal_id) ) {
 				# Only save if no error
 				$animeStats->save();
 				$counter++;
@@ -242,6 +242,7 @@ class Background {
 			$enhancement = $anime->getEnhancementFromAnilist();
 		} catch( \Exception $e ) {
 			echo "addEnhancementToAnime: Getting Data from Anilist failed for anime: " . $anime->title_pref . " (" . $anime->mal_id . ")\n";
+			echo $e->getMessage() ."\n";
 			return;
 		}
 		if ( "" == $enhancement ) {
@@ -251,6 +252,15 @@ class Background {
 		$update = array();
 		if ( ! is_null( $enhancement->description ) ) {
 			$update["synopsis"] = str_replace('<br>', '%%br%%', $enhancement->description);
+
+			/*
+			 * Fix broken html
+			 */
+			$tidy = new \Tidy();
+			$tidy->parseString($update["synopsis"], [ 'show-body-only' => true ], 'utf8');
+			$tidy->cleanRepair();
+			$update["synopsis"] = $tidy;
+
 		}
 		if ( ! is_null( $enhancement->hashtag ) ) {
 			$update["hashtag"] = $enhancement->hashtag;
@@ -269,14 +279,18 @@ class Background {
 		DB::table('anime')
 			->where('mal_id', $anime->mal_id)
 			->update( $update );
+
+		sleep(1);
 	}
 
 	public function saveEnhancementForAll() {
 		#$anime_all = Anime::where('airing_status', env('ANIME_IS_AIRING'))->where('synopsis', '')->get();
 		$anime_all = Anime::where('synopsis', '')->orWhereNull('duration')->get();
+		#$anime_all = Anime::where('synopsis', '!=', '')->get();
 		foreach( $anime_all as $anime ) {
 			$this->addEnhancementToAnime($anime);
-			sleep(1);
+			#usleep(500000); # 0.5s
+			sleep(5);
 		}
 	}
 
@@ -335,7 +349,7 @@ class Background {
 			/**
 			 * Sleep to avoid 403 by MAL.
 			 */
-			$anime->fill( $entry->getMalID() );
+			$anime->fillStats( $entry->getMalID() );
 
 			if ( "" == $anime->url ) {
 				echo $caller . ": url is empty for: " . $entry->getMalId() . "\n";
@@ -401,7 +415,7 @@ class Background {
 		foreach( $anime as $entry ) {
 			$count++;
 			$fill = new Anime;
-			$fill->fill( $entry->mal_id, $skip_if_unpopular=false ); # check every anime we have, even with less than 10k members
+			$fill->fillStats( $entry->mal_id, $skip_if_unpopular=false ); # check every anime we have, even with less than 10k members
 
 			if ( $fill->image_url != $entry->image_url ) {
 				echo "Image URL (" . $entry->mal_id . ", " . $entry->title_pref . ") changed from " . $entry->image_url . " to " . $fill->image_url . "\n";
diff --git a/app/MALUser.php b/app/MALUser.php
index f49d09a..4b781f8 100644
--- a/app/MALUser.php
+++ b/app/MALUser.php
@@ -8,7 +8,7 @@ use Jikan\MyAnimeList\MalClient;
 class MALUser extends Model
 {
 	protected $table = 'malusers';
-	protected $fillable = 'username';
+	protected $fillable = [ 'username' ];
 
 	public function __construct() {
 	}