# Install necessary dependency packages
pip install requests beautifulsoup4 selenium lxml fake-useragent
pip install pandas numpy matplotlib seaborn
pip install requests-html selenium-wire

import requests
from bs4 import BeautifulSoup
import time
import random
from fake_useragent import UserAgent
import json
import pandas as pd
from urllib.parse import urljoin, urlparse
import logging

class AmazonReviewScraper:
    def __init__(self):
        self.session = requests.Session()
        self.ua = UserAgent()
        self.setup_headers()
        self.setup_logging()
        
    def setup_headers(self):
        """Set request headers to simulate real browser behavior"""
        self.headers = {
            'User-Agent': self.ua.random,
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
            'Accept-Encoding': 'gzip, deflate, br',
            'Connection': 'keep-alive',
            'Upgrade-Insecure-Requests': '1',
            'Sec-Fetch-Dest': 'document',
            'Sec-Fetch-Mode': 'navigate',
            'Sec-Fetch-Site': 'none',
        }
        self.session.headers.update(self.headers)
    
    def setup_logging(self):
        """Configure the logging system"""
        logging.basicConfig(
            level=logging.INFO,
            format='%(asctime)s - %(levelname)s - %(message)s',
            handlers=[
                logging.FileHandler('amazon_scraper.log'),
                logging.StreamHandler()
            ]
        )
        self.logger = logging.getLogger(__name__)
    
    def get_product_reviews(self, product_url, max_pages=5):
        """Get product review data"""
        try:
            # Parse the product ID (ASIN)
            asin = self.extract_asin(product_url)
            if not asin:
                self.logger.error(f"Could not extract ASIN from: {product_url}")
                return []
            
            reviews = []
            for page in range(1, max_pages + 1):
                page_reviews = self.scrape_review_page(asin, page)
                if not page_reviews:
                    break
                reviews.extend(page_reviews)
                
                # Random delay to avoid detection
                time.sleep(random.uniform(2, 5))
            
            return reviews
            
        except Exception as e:
            self.logger.error(f"Failed to get reviews: {e}")
            return []
    
    def extract_asin(self, url):
        """Extract ASIN from the URL"""
        # ASIN extraction for multiple URL formats
        patterns = [
            r'/dp/([A-Z09]{10})',
            r'/product/([A-Z09]{10})',
            r'asin=([A-Z09]{10})',
        ]
        
        for pattern in patterns:
            import re
            match = re.search(pattern, url)
            if match:
                return match.group(1)
        return None
    
    def scrape_review_page(self, asin, page_num):
        """Scrape reviews from a specific page"""
        review_url = f"https://www.amazon.com/product-reviews/{asin}"
        params = {
            'pageNumber': page_num,
            'sortBy': 'recent'
        }
        
        try:
            response = self.session.get(review_url, params=params, timeout=10)
            response.raise_for_status()
            
            soup = BeautifulSoup(response.content, 'html.parser')
            return self.parse_reviews(soup)
            
        except requests.RequestException as e:
            self.logger.error(f"Request failed - page {page_num}: {e}")
            return []
    
    def parse_reviews(self, soup):
        """Parse review data"""
        reviews = []
        review_elements = soup.find_all('div', {'data-hook': 'review'})
        
        for element in review_elements:
            try:
                review = self.extract_review_data(element)
                if review:
                    reviews.append(review)
            except Exception as e:
                self.logger.warning(f"Failed to parse a single review: {e}")
                continue
        
        return reviews
    
    def extract_review_data(self, element):
        """Extract detailed information from a single review"""
        review = {}
        
        # Review title
        title_elem = element.find('a', {'data-hook': 'review-title'})
        review['title'] = title_elem.get_text(strip=True) if title_elem else ''
        
        # Rating
        rating_elem = element.find('i', {'data-hook': 'review-star-rating'})
        if rating_elem:
            rating_text = rating_elem.get_text(strip=True)
            rating = rating_text.split()[0] if rating_text else '0'
            review['rating'] = float(rating)
        
        # Review content
        content_elem = element.find('span', {'data-hook': 'review-body'})
        review['content'] = content_elem.get_text(strip=True) if content_elem else ''
        
        # Review date
        date_elem = element.find('span', {'data-hook': 'review-date'})
        review['date'] = date_elem.get_text(strip=True) if date_elem else ''
        
        # Username
        author_elem = element.find('span', class_='a-profile-name')
        review['author'] = author_elem.get_text(strip=True) if author_elem else ''
        
        # Helpfulness vote
        helpful_elem = element.find('span', {'data-hook': 'helpful-vote-statement'})
        review['helpful_votes'] = helpful_elem.get_text(strip=True) if helpful_elem else '0'
        
        return review

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

class AdvancedAmazonScraper(AmazonReviewScraper):
    def __init__(self, use_selenium=False, proxy_list=None):
        super().__init__()
        self.use_selenium = use_selenium
        self.proxy_list = proxy_list or []
        self.current_proxy_index = 0
        
        if use_selenium:
            self.setup_selenium()
    
    def setup_selenium(self):
        """Configure Selenium WebDriver"""
        chrome_options = Options()
        chrome_options.add_argument('--no-sandbox')
        chrome_options.add_argument('--disable-dev-shm-usage')
        chrome_options.add_argument('--disable-blink-features=AutomationControlled')
        chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
        chrome_options.add_experimental_option('useAutomationExtension', False)
        
        # Random user agent
        chrome_options.add_argument(f'--user-agent={self.ua.random}')
        
        # Configure proxy
        if self.proxy_list:
            proxy = self.get_next_proxy()
            chrome_options.add_argument(f'--proxy-server={proxy}')
        
        self.driver = webdriver.Chrome(options=chrome_options)
        self.driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
    
    def get_next_proxy(self):
        """Rotate proxy IPs"""
        if not self.proxy_list:
            return None
        
        proxy = self.proxy_list[self.current_proxy_index]
        self.current_proxy_index = (self.current_proxy_index + 1) % len(self.proxy_list)
        return proxy
    
    def handle_captcha(self, driver):
        """Handle CAPTCHA (requires manual intervention or a third-party service)"""
        try:
            # Check if a CAPTCHA is present
            captcha_element = driver.find_element(By.ID, "captchacharacters")
            if captcha_element:
                self.logger.warning("CAPTCHA detected, manual intervention required")
                # CAPTCHA solving service can be integrated here
                input("Please solve the CAPTCHA manually and press Enter to continue...")
                return True
        except:
            pass
        return False
    
    def scrape_with_selenium(self, product_url):
        """Scrape using Selenium (can handle JavaScript rendering)"""
        try:
            self.driver.get(product_url)
            
            # Handle potential CAPTCHAs
            self.handle_captcha(self.driver)
            
            # Wait for the page to load completely
            WebDriverWait(self.driver, 10).until(
                EC.presence_of_element_located((By.ID, "reviewsMedley"))
            )
            
            # Scroll the page to trigger lazy loading
            self.scroll_to_load_reviews()
            
            # Parse the page content
            soup = BeautifulSoup(self.driver.page_source, 'html.parser')
            return self.parse_reviews(soup)
            
        except Exception as e:
            self.logger.error(f"Selenium scraping failed: {e}")
            return []
    
    def scroll_to_load_reviews(self):
        """Scroll the page to load more reviews"""
        last_height = self.driver.execute_script("return document.body.scrollHeight")
        
        while True:
            # Scroll to the bottom of the page
            self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            
            # Wait for the page to load
            time.sleep(2)
            
            # Check if new content has loaded
            new_height = self.driver.execute_script("return document.body.scrollHeight")
            if new_height == last_height:
                break
            last_height = new_height

import itertools
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry

class ProxyManager:
    def __init__(self, proxy_list):
        self.proxy_list = proxy_list
        self.proxy_cycle = itertools.cycle(proxy_list)
        self.failed_proxies = set()
    
    def get_working_proxy(self):
        """Get a working proxy"""
        for _ in range(len(self.proxy_list)):
            proxy = next(self.proxy_cycle)
            if proxy not in self.failed_proxies:
                if self.test_proxy(proxy):
                    return proxy
                else:
                    self.failed_proxies.add(proxy)
        return None
    
    def test_proxy(self, proxy):
        """Test proxy availability"""
        try:
            response = requests.get(
                'http://httpbin.org/ip',
                proxies={'http': proxy, 'https': proxy},
                timeout=5
            )
            return response.status_code == 200
        except:
            return False

class EnhancedAmazonScraper(AdvancedAmazonScraper):
    def __init__(self, proxy_list=None):
        super().__init__()
        self.proxy_manager = ProxyManager(proxy_list) if proxy_list else None
        self.setup_session_retry()
    
    def setup_session_retry(self):
        """Configure a retry mechanism"""
        retry_strategy = Retry(
            total=3,
            backoff_factor=1,
            status_forcelist=[429, 500, 502, 503, 504],
        )
        adapter = HTTPAdapter(max_retries=retry_strategy)
        self.session.mount("http://", adapter)
        self.session.mount("https://", adapter)
    
    def make_request_with_proxy(self, url, **kwargs):
        """Send a request using a proxy"""
        if self.proxy_manager:
            proxy = self.proxy_manager.get_working_proxy()
            if proxy:
                kwargs['proxies'] = {'http': proxy, 'https': proxy}
        
        return self.session.get(url, **kwargs)

import threading
from collections import defaultdict
from datetime import datetime, timedelta

class RateLimiter:
    def __init__(self, max_requests_per_minute=30):
        self.max_requests = max_requests_per_minute
        self.requests = defaultdict(list)
        self.lock = threading.Lock()
    
    def wait_if_needed(self, domain='amazon.com'):
        """Limit request frequency based on the domain"""
        with self.lock:
            now = datetime.now()
            minute_ago = now - timedelta(minutes=1)
            
            # Clear expired records
            self.requests[domain] = [
                req_time for req_time in self.requests[domain]
                if req_time > minute_ago
            ]
            
            # Check if waiting is necessary
            if len(self.requests[domain]) >= self.max_requests:
                oldest_request = min(self.requests[domain])
                wait_time = 60 - (now - oldest_request).total_seconds()
                if wait_time > 0:
                    time.sleep(wait_time)
            
            # Record the current request
            self.requests[domain].append(now)

class LimitedReviewAnalyzer:
    def __init__(self):
        self.sentiment_keywords = {
            'positive': ['excellent', 'amazing', 'great', 'perfect', 'love', 'awesome'],
            'negative': ['terrible', 'awful', 'hate', 'worst', 'horrible', 'disappointing']
        }
    
    def extract_accessible_reviews(self, product_url):
        """Extract accessible review data (usually 8-10 reviews)"""
        scraper = EnhancedAmazonScraper()
        
        try:
            response = scraper.session.get(product_url)
            soup = BeautifulSoup(response.content, 'html.parser')
            
            # Find the review section
            review_section = soup.find('div', {'id': 'reviews-medley-footer'})
            if not review_section:
                review_section = soup.find('div', {'data-hook': 'reviews-medley-footer'})
            
            reviews = []
            if review_section:
                review_elements = review_section.find_all('div', {'data-hook': 'review'})
                
                for element in review_elements:
                    review = scraper.extract_review_data(element)
                    if review:
                        reviews.append(review)
            
            return reviews
            
        except Exception as e:
            logging.error(f"Failed to extract reviews: {e}")
            return []
    
    def analyze_customer_says(self, product_url):
        """Analyze Customer Says data"""
        scraper = EnhancedAmazonScraper()
        
        try:
            response = scraper.session.get(product_url)
            soup = BeautifulSoup(response.content, 'html.parser')
            
            # Find the Customer Says section
            customer_says = soup.find('div', {'data-hook': 'cr-insights-widget'})
            if customer_says:
                insights = []
                insight_elements = customer_says.find_all('div', class_='cr-insights-text')
                
                for element in insight_elements:
                    text = element.get_text(strip=True)
                    sentiment = self.analyze_sentiment(text)
                    insights.append({
                        'text': text,
                        'sentiment': sentiment
                    })
                
                return insights
            
        except Exception as e:
            logging.error(f"Failed to analyze Customer Says: {e}")
        
        return []
    
    def analyze_sentiment(self, text):
        """Simple sentiment analysis"""
        text_lower = text.lower()
        positive_count = sum(1 for word in self.sentiment_keywords['positive'] if word in text_lower)
        negative_count = sum(1 for word in self.sentiment_keywords['negative'] if word in text_lower)
        
        if positive_count > negative_count:
            return 'positive'
        elif negative_count > positive_count:
            return 'negative'
        else:
            return 'neutral'

import requests
import json
import time

class PangolinAmazonReviewAPI:
    def __init__(self, api_key):
        self.api_key = api_key
        self.base_url = "https://scrapeapi.pangolinfo.com/api/v1/scrape"
        self.headers = {
            "Authorization": f"Bearer {api_key}",
            "Content-Type": "application/json"
        }
    
    def scrape_product_details_with_reviews(self, product_url, zipcode="10041"):
        """
        Get the product detail page, including complete review data
        
        Args:
            product_url: Amazon product page URL
            zipcode: Specify a zip code, which affects price and shipping info
        
        Returns:
            A complete response containing product information and review data
        """
        payload = {
            "url": product_url,
            "formats": ["json"],  # Get structured data
            "parserName": "amzProductDetail",  # Use the Amazon product detail parser
            "bizContext": {
                "zipcode": zipcode  # Zip code can be adjusted as needed
            }
        }
        
        try:
            response = requests.post(
                self.base_url, 
                json=payload, 
                headers=self.headers,
                timeout=30
            )
            response.raise_for_status()
            
            result = response.json()
            if result['code'] == 0:
                return self.parse_review_data(result['data'])
            else:
                print(f"API call failed: {result.get('message', 'Unknown error')}")
                return None
                
        except requests.exceptions.RequestException as e:
            print(f"Request failed: {e}")
            return None
    
    def parse_review_data(self, api_response):
        """Parse the review data returned by the API"""
        parsed_data = {
            'product_info': {},
            'reviews': [],
            'customer_says': [],
            'review_summary': {}
        }
        
        # Parse basic product information
        if 'product' in api_response:
            product = api_response['product']
            parsed_data['product_info'] = {
                'title': product.get('title', ''),
                'asin': product.get('asin', ''),
                'brand': product.get('brand', ''),
                'price': product.get('price', ''),
                'rating': product.get('rating', ''),
                'review_count': product.get('reviewCount', 0)
            }
        
        # Parse review data
        if 'reviews' in api_response:
            for review in api_response['reviews']:
                parsed_data['reviews'].append({
                    'id': review.get('id', ''),
                    'title': review.get('title', ''),
                    'content': review.get('content', ''),
                    'rating': review.get('rating', 0),
                    'author': review.get('author', ''),
                    'date': review.get('date', ''),
                    'helpful_votes': review.get('helpfulVotes', 0),
                    'verified_purchase': review.get('verifiedPurchase', False)
                })
        
        # Parse Customer Says data (a special feature of the Pangolin API)
        if 'customerSays' in api_response:
            parsed_data['customer_says'] = api_response['customerSays']
        
        # Parse review summary statistics
        if 'reviewSummary' in api_response:
            parsed_data['review_summary'] = api_response['reviewSummary']
        
        return parsed_data
    
    def batch_scrape_reviews(self, product_urls, batch_size=10, delay=1):
        """
        Batch scrape review data for multiple products
        
        Args:
            product_urls: A list of product URLs
            batch_size: The batch processing size
            delay: The interval between requests (in seconds)
        """
        all_results = []
        
        for i in range(0, len(product_urls), batch_size):
            batch = product_urls[i:i + batch_size]
            batch_results = []
            
            for url in batch:
                print(f"Processing: {url}")
                result = self.scrape_product_details_with_reviews(url)
                if result:
                    batch_results.append(result)
                
                # Add a delay to avoid overly frequent requests
                time.sleep(delay)
            
            all_results.extend(batch_results)
            print(f"Completed batch {i//batch_size + 1}/{(len(product_urls)-1)//batch_size + 1}")
        
        return all_results

# Usage example
def demo_pangolin_api():
    """Demonstrate how to use the Pangolin API to get Amazon review data"""
    
    # Initialize the API client (replace with your real API Key)
    api_client = PangolinAmazonReviewAPI("your-api-key-here")
    
    # Example product URLs
    test_urls = [
        "https://www.amazon.com/dp/B0DYTF8L2W",
        "https://www.amazon.com/dp/B08N5WRWNW",
        "https://www.amazon.com/dp/B07YN2ZZQC"
    ]
    
    # Get data for a single product
    single_result = api_client.scrape_product_details_with_reviews(test_urls[0])
    
    if single_result:
        print("=== Product Information ===")
        print(json.dumps(single_result['product_info'], indent=2))
        
        print("\n=== Review Data ===")
        for i, review in enumerate(single_result['reviews'][:3]):  # Show the first 3 reviews
            print(f"Review {i+1}:")
            print(f"  Title: {review['title']}")
            print(f"  Rating: {review['rating']}")
            print(f"  Content: {review['content'][:100]}...")
            print(f"  Author: {review['author']}")
            print(f"  Date: {review['date']}")
            print()
        
        print("=== Customer Says Insights ===")
        if single_result['customer_says']:
            for insight in single_result['customer_says'][:5]:  # Show the first 5 insights
                print(f"- {insight}")
        
    # Batch processing example
    print("\n=== Batch Processing Example ===")
    batch_results = api_client.batch_scrape_reviews(test_urls[:2], batch_size=2)
    print(f"Successfully retrieved data for {len(batch_results)} products")

if __name__ == "__main__":
    demo_pangolin_api()

def analyze_customer_says_data(customer_says_data):
    """
    Analyze Customer Says data to extract key insights
    """
    insights = {
        'positive_aspects': [],
        'negative_aspects': [],
        'feature_mentions': {},
        'sentiment_distribution': {'positive': 0, 'negative': 0, 'neutral': 0}
    }
    
    for item in customer_says_data:
        # Classify positive and negative feedback
        if item.get('sentiment') == 'positive':
            insights['positive_aspects'].append(item['text'])
            insights['sentiment_distribution']['positive'] += 1
        elif item.get('sentiment') == 'negative':
            insights['negative_aspects'].append(item['text'])
            insights['sentiment_distribution']['negative'] += 1
        else:
            insights['sentiment_distribution']['neutral'] += 1
        
        # Extract feature mentions
        if 'feature' in item:
            feature = item['feature']
            if feature not in insights['feature_mentions']:
                insights['feature_mentions'][feature] = {'count': 0, 'sentiment': []}
            insights['feature_mentions'][feature]['count'] += 1
            insights['feature_mentions'][feature]['sentiment'].append(item.get('sentiment', 'neutral'))
    
    return insights

class ComprehensiveReviewAnalysis:
    def __init__(self, api_client):
        self.api_client = api_client
    
    def get_complete_review_analysis(self, product_url):
        """Get a complete review analysis report"""
        
        # Use the Pangolin API to get complete data
        raw_data = self.api_client.scrape_product_details_with_reviews(product_url)
        
        if not raw_data:
            return None
        
        analysis_report = {
            'product_overview': self.analyze_product_overview(raw_data['product_info']),
            'review_statistics': self.calculate_review_statistics(raw_data['reviews']),
            'sentiment_analysis': self.perform_sentiment_analysis(raw_data['reviews']),
            'keyword_analysis': self.extract_keywords(raw_data['reviews']),
            'customer_insights': self.analyze_customer_says_data(raw_data['customer_says']),
            'competitive_positioning': self.analyze_competitive_position(raw_data)
        }
        
        return analysis_report
    
    def analyze_product_overview(self, product_info):
        """Analyze the product overview"""
        return {
            'title': product_info['title'],
            'overall_rating': product_info['rating'],
            'total_reviews': product_info['review_count'],
            'price_point': product_info['price'],
            'brand_reputation': product_info['brand']
        }
    
    def calculate_review_statistics(self, reviews):
        """Calculate review statistics"""
        if not reviews:
            return {}
        
        ratings = [review['rating'] for review in reviews if review['rating']]
        
        return {
            'total_reviews': len(reviews),
            'average_rating': sum(ratings) / len(ratings) if ratings else 0,
            'rating_distribution': self.get_rating_distribution(ratings),
            'verified_purchase_ratio': len([r for r in reviews if r.get('verified_purchase')]) / len(reviews),
            'average_review_length': sum(len(r['content']) for r in reviews) / len(reviews)
        }
    
    def get_rating_distribution(self, ratings):
        """Get the rating distribution"""
        distribution = {1: 0, 2: 0, 3: 0, 4: 0, 5: 0}
        for rating in ratings:
            if rating in distribution:
                distribution[int(rating)] += 1
        return distribution
    
    def perform_sentiment_analysis(self, reviews):
        """Perform sentiment analysis"""
        sentiments = {'positive': 0, 'negative': 0, 'neutral': 0}
        
        positive_keywords = ['excellent', 'amazing', 'great', 'perfect', 'love', 'awesome', 'fantastic', 'wonderful']
        negative_keywords = ['terrible', 'awful', 'hate', 'worst', 'horrible', 'disappointing', 'useless', 'broken']
        
        for review in reviews:
            content_lower = review['content'].lower()
            positive_count = sum(1 for word in positive_keywords if word in content_lower)
            negative_count = sum(1 for word in negative_keywords if word in content_lower)
            
            if positive_count > negative_count:
                sentiments['positive'] += 1
            elif negative_count > positive_count:
                sentiments['negative'] += 1
            else:
                sentiments['neutral'] += 1
        
        return sentiments
    
    def extract_keywords(self, reviews):
        """Extract keywords"""
        from collections import Counter
        import re
        
        # Combine all review content
        all_text = ' '.join([review['content'] for review in reviews])
        
        # Simple word extraction (more complex NLP techniques can be used in practice)
        words = re.findall(r'\b[a-zA-Z]{3,}\b', all_text.lower())
        
        # Filter common stop words
        stop_words = {'the', 'and', 'for', 'are', 'but', 'not', 'you', 'all', 'can', 'had', 'have', 'has', 'what'}
        filtered_words = [word for word in words if word not in stop_words]
        
        # Get the 20 most common words
        return dict(Counter(filtered_words).most_common(20))