From 19e0c104bd52428780b5e2072d1cd88fd12e7da1 Mon Sep 17 00:00:00 2001
From: horus
Date: Thu, 23 Jul 2020 15:39:40 +0200
Subject: Bugfix. Crawler starts indexing anime stats in the beginning of the
 day, not at the end.

---
 app/Console/Kernel.php       |  3 ++-
 app/Libraries/Background.php | 16 ++++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php
index 3bd744e..f7dc4eb 100644
--- a/app/Console/Kernel.php
+++ b/app/Console/Kernel.php
@@ -36,7 +36,8 @@ class Kernel extends ConsoleKernel
 	$schedule->call( function(){
 		$background = new Background();
 		$background->saveAnimeStats();
-	})->dailyAt('22:00');
+	})->everyThirtyMinutes()->name('saveStats')->withoutOverlapping(2);
+	#})->everyMinute();
 
 	$schedule->call( function(){
 		$background = new Background();
diff --git a/app/Libraries/Background.php b/app/Libraries/Background.php
index c491d44..920c866 100644
--- a/app/Libraries/Background.php
+++ b/app/Libraries/Background.php
@@ -123,6 +123,8 @@ class Background {
 		        ))
 		);
 
+		sleep(10);
+
 		foreach($season->anime as $entry) {
 			if ( Anime::where('mal_id', $entry->getMalID())->exists() ) {
 				/**
@@ -148,6 +150,7 @@ class Background {
 			if( ! DB::table('anime')->where('mal_id', $entry->getMalID() )->exists() ) {
 				$anime->save();
 			}
+			sleep(10);
 		}
 	}
 
@@ -157,10 +160,22 @@ class Background {
 		}
 
 		foreach($anime as $entry ) {
+			# skip if we have crawled stats from today
+			if ( ! empty($entry->getStats()->get()->last()) ) {
+				$date = $entry->getStats()->get()->last()->created_at->toDateTimeString();
+				if (date('Ymd') == date('Ymd', strtotime($date)) ) {
+					continue;
+				}
+			}
+
+			echo "Getting stats for " . $entry->mal_id ."\n";
 			$animeStats = new AnimeStats();
 			$animeStats->fill( $entry->mal_id );
 
 			$animeStats->save();
+
+			# sleep to avoid 403
+			sleep(5);
 		}
 	}
 
@@ -176,6 +191,7 @@ class Background {
 				echo "checkIfIsAiring: Problem requesting AnimeInfo for (" . $anime->mal_id . ") continue\n";
 				echo $e->getMessage();
 				echo "\n\n";
+				continue;
 			}
 			
 			if ( ! $animeInfo->isAiring() ) {
-- 
cgit v1.2.3