# -*- coding: utf-8 -*-
import sys
import os, re, json, hashlib, tldextract, logging, random, time
from datetime import datetime, time as dt_time

import pandas as pd
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
from tqdm import tqdm

# ===================== CONFIG =====================
INPUT_EXCEL = "input_siteuri.xlsx"
OUTPUT_DIR = "/home4/etimpuls/catalog-output"
   # ← modifica la tine (recomand slash-uri '/')
USER_AGENT = (
    "Mozilla/5.0 (compatible; "
    "eTimpulSEO-ProductDataBot/1.3; "
    "+https://etimpulseo.ro; "
    "role=seo-technical-audit; "
    "scope=factual-product-data-only; "
    "behavior=low-frequency-low-impact; "
    "contact=contact@etimpulseo.ro)"
)

# delay "uman" intre request-uri
MIN_DELAY   = 5.0    # secunde - pauza minima
MAX_DELAY   = 15.0   # secunde - pauza maxima

MAX_PAGES   = 2000
TIMEOUT     = 20
SAVE_EVERY  = 25  # salveaza intermediar la fiecare N produse
# ==================================================

HEADERS = {"User-Agent": USER_AGENT}
os.makedirs(OUTPUT_DIR, exist_ok=True)
IMAGES_DIR = os.path.join(OUTPUT_DIR, "imagini")
os.makedirs(IMAGES_DIR, exist_ok=True)

# logging in fisier
logging.basicConfig(
    filename=os.path.join(OUTPUT_DIR, "scraper.log"),
    level=logging.INFO,
    format="%(asctime)s %(levelname)s: %(message)s",
)

PRODUCT_URL_HINTS = ["/produs/","/product/","/produse/","/products/","/item/"]

# ===================== PROGRAM DE RULARE =====================
def is_allowed_time():
    """
    Verifica daca scraperul are voie sa ruleze in functie de zi si ora.

    Luni–Vineri:   07:00 – 22:00
    Sambata–Duminica: 09:00 – 18:00
    """
    now = datetime.now()
    current_time = now.time()
    weekday = now.weekday()  # 0 = Luni, 6 = Duminica

    # Luni–Vineri: 07:00–22:00
    if 0 <= weekday <= 4:
        return dt_time(7, 0) <= current_time <= dt_time(22, 0)

    # Sambata–Duminica: 09:00–18:00
    if weekday in (5, 6):
        return dt_time(9, 0) <= current_time <= dt_time(18, 0)

    return False


def wait_until_allowed():
    """
    Daca scraperul este in afara intervalului permis, asteapta automat.
    Verifica la fiecare 5 minute daca a intrat in program.
    """
    while not is_allowed_time():
        print("⏸ Pauza automata: in afara programului de scraping. Reincerc in 5 minute...")
        time.sleep(300)  # 5 minute
# ============================================================


def human_sleep():
    """
    Pauza pseudo-umana intre request-uri:
    - de obicei intre MIN_DELAY si MAX_DELAY
    - ocazional o pauza mai lunga (20-60s), ca un "break"
    """
    # pauza normala
    delay = random.uniform(MIN_DELAY, MAX_DELAY)

    # ~5% din cazuri: pauza mai lunga, ca un mic "break"
    if random.random() < 0.05:
        extra = random.uniform(20.0, 60.0)
        delay += extra
        logging.info(f"Pauza lunga (break): {delay:.1f} secunde")

    time.sleep(delay)

def same_domain(url_a, url_b):
    a = tldextract.extract(url_a)
    b = tldextract.extract(url_b)
    return (a.domain, a.suffix) == (b.domain, b.suffix)

def fetch(url):
    r = requests.get(url, headers=HEADERS, timeout=TIMEOUT)
    r.raise_for_status()
    return r.text

def is_product_page(html):
    try:
        soup = BeautifulSoup(html, "lxml")
        for tag in soup.find_all("script", attrs={"type": "application/ld+json"}):
            try:
                data = json.loads(tag.string or "")
                if isinstance(data, list):
                    arr = data
                else:
                    arr = [data]
                for node in arr:
                    if isinstance(node, dict) and node.get("@type") in ["Product","product"]:
                        return True
            except Exception:
                continue
    except Exception:
        pass
    return False

def clean_text(text):
    return re.sub(r"\s+", " ", text or "").strip()

def find_ean_in_html(soup):
    for itemprop in ["gtin13","gtin14","gtin12","gtin8"]:
        tag = soup.select_one(f'[itemprop="{itemprop}"]')
        if tag:
            val = tag.get("content") or tag.get_text(strip=True)
            if val and re.search(r"\d{8,14}", val):
                return re.search(r"\d{8,14}", val).group(0)
    text = soup.get_text(" ").lower()
    m = re.search(r"(ean|gtin)[^\d]{0,20}(\d{8,14})", text)
    if m:
        return m.group(2)
    m = re.search(r"\b\d{13}\b", text)
    if m:
        return m.group(0)
    return None

def parse_product(url, html):
    soup = BeautifulSoup(html, "lxml")
    ld = {}
    for tag in soup.find_all("script", {"type": "application/ld+json"}):
        try:
            data = json.loads(tag.string or "")
            if isinstance(data, list):
                for n in data:
                    if isinstance(n, dict) and n.get("@type") in ["Product","product"]:
                        ld = n
                        break
            elif isinstance(data, dict) and data.get("@type") in ["Product","product"]:
                ld = data
        except Exception:
            continue
        if ld:
            break

    name = (ld.get("name") if isinstance(ld, dict) else None) or \
           (soup.select_one("h1, .product_title") and soup.select_one("h1, .product_title").get_text(strip=True))

    ean = None
    if isinstance(ld, dict):
        for key in ["gtin13","gtin14","gtin12","gtin8"]:
            if ld.get(key):
                m = re.search(r"\d{8,14}", str(ld.get(key)))
                if m:
                    ean = m.group(0)
                    break
        if not ean and str(ld.get("sku","")).isdigit() and 8 <= len(str(ld.get("sku"))) <= 14:
            ean = ld.get("sku")
    if not ean:
        ean = find_ean_in_html(soup)

    desc = None
    if isinstance(ld, dict) and ld.get("description"):
        desc = clean_text(ld.get("description"))
    if not desc:
        for sel in ["#tab-description",".woocommerce-Tabs-panel--description",
                    ".product-short-description",".description",".entry-content"]:
            el = soup.select_one(sel)
            if el and clean_text(el.get_text()):
                desc = clean_text(el.get_text())
                break

    image_url = None
    if isinstance(ld, dict):
        img = ld.get("image")
        if isinstance(img, list) and img:
            image_url = img[0]
        elif isinstance(img, str):
            image_url = img
    if not image_url:
        gal = soup.select_one(".woocommerce-product-gallery img, .product-gallery img, img")
        if gal and gal.get("src"):
            image_url = gal.get("src")

    return {
        "URL": url,
        "Nume produs": clean_text(name),
        "EAN": ean,
        "Descriere produs": desc,
        "Imagine produs (URL)": image_url
    }

def download_image(img_url, folder):
    try:
        r = requests.get(img_url, headers=HEADERS, timeout=TIMEOUT, stream=True)
        r.raise_for_status()
        ext = ".jpg"
        ct = r.headers.get("Content-Type","").lower()
        if "png" in ct or img_url.lower().endswith(".png"):
            ext = ".png"
        h = hashlib.md5(img_url.encode("utf-8")).hexdigest()[:12] + ext
        out = os.path.join(folder, h)
        with open(out, "wb") as f:
            for chunk in r.iter_content(8192):
                f.write(chunk)
        return out
    except Exception as e:
        logging.warning(f"Imagine esuata {img_url}: {e}")
        return None

def crawl_site(start_url):
    visited = set()
    to_visit = [start_url]
    product_urls = set()
    pages = 0

    base = "{u.scheme}://{u.netloc}".format(u=urlparse(start_url))
    print(f"• Domeniu: {base}")

    while to_visit and pages < MAX_PAGES:
        # respecta programul de rulare
        wait_until_allowed()

        url = to_visit.pop(0)
        if url in visited:
            continue
        visited.add(url)
        pages += 1

        print(f"[{pages}] Vizitez: {url} | Coada: {len(to_visit)} | Produse gasite: {len(product_urls)}")
        logging.info(f"Visiting {url}")

        try:
            html = fetch(url)
        except Exception as e:
            print("Eroare:", e)
            logging.error(f"Fetch error {url}: {e}")
            continue

        if is_product_page(html) or any(h in url for h in PRODUCT_URL_HINTS):
            product_urls.add(url)
        else:
            soup = BeautifulSoup(html, "lxml")
            for a in soup.select("a[href]"):
                href = a["href"]
                abs_url = urljoin(url, href)
                if abs_url.startswith(base) and abs_url not in visited:
                    if not re.search(r"\.(jpg|jpeg|png|gif|svg|pdf|zip|rar)(\?|$)", abs_url, re.I) and "#" not in abs_url:
                        to_visit.append(abs_url)

        human_sleep()

    return list(product_urls), pages

def main():
    all_rows = []
    total_pages_visited = 0
    errors = 0

    # daca a fost transmis un URL in linia de comanda, lucram doar cu el
    if len(sys.argv) > 1:
        start_url = sys.argv[1].strip()
        if not start_url.startswith("http"):
            print("URL invalid:", start_url)
            return

        print("\n=== Pornesc crawl pentru (CLI):", start_url)
        product_urls, pages = crawl_site(start_url)
        total_pages_visited += pages
        print(f"\n✔ Gasit {len(product_urls)} pagini de produs (din {pages} pagini vizitate).")

        pbar = tqdm(product_urls, desc="Parsam produse", unit="produs")
        parsed_count = 0
        with_ean = 0

        for purl in pbar:
            # respecta programul de rulare si aici
            wait_until_allowed()

            try:
                html = fetch(purl)
                data = parse_product(purl, html)

                local_img = download_image(
                    data.get("Imagine produs (URL)"),
                    os.path.join(OUTPUT_DIR, "imagini")
                ) if data.get("Imagine produs (URL)") else None

                data["Imagine produs (LOCAL)"] = local_img

                if data.get("EAN"):
                    with_ean += 1
                all_rows.append(data)
                parsed_count += 1

                pbar.set_postfix({"total": parsed_count, "cu_EAN": with_ean})

            except Exception as e:
                errors += 1
                logging.error(f"Eroare produs {purl}: {e}")

            human_sleep()

    else:
        # comportamentul vechi: citeste lista din Excel
        excel = pd.read_excel(INPUT_EXCEL)

        for _, row in excel.iterrows():
            start_url = str(row["URL_start"]).strip()
            if not start_url or not start_url.startswith("http"):
                print("Sari peste URL invalid:", start_url)
                continue

            print("\n=== Pornesc crawl pentru:", start_url)
            product_urls, pages = crawl_site(start_url)
            total_pages_visited += pages
            print(f"\n✔ Gasit {len(product_urls)} pagini de produs (din {pages} pagini vizitate).")

            pbar = tqdm(product_urls, desc="Parsam produse", unit="produs")
            parsed_count = 0
            with_ean = 0

            for purl in pbar:
                # respecta programul de rulare si aici
                wait_until_allowed()

                try:
                    html = fetch(purl)
                    data = parse_product(purl, html)

                    local_img = download_image(
                        data.get("Imagine produs (URL)"),
                        os.path.join(OUTPUT_DIR, "imagini")
                    ) if data.get("Imagine produs (URL)") else None

                    data["Imagine produs (LOCAL)"] = local_img

                    if data.get("EAN"):
                        with_ean += 1
                    all_rows.append(data)
                    parsed_count += 1

                    pbar.set_postfix({"total": parsed_count, "cu_EAN": with_ean})

                    if parsed_count % SAVE_EVERY == 0:
                        out_xlsx = os.path.join(OUTPUT_DIR, "produse_extrase_partial.xlsx")
                        pd.DataFrame(all_rows).to_excel(out_xlsx, index=False)

                except Exception as e:
                    errors += 1
                    logging.error(f"Eroare produs {purl}: {e}")

                human_sleep()

    # salvare finala
    if all_rows:
        out_xlsx = os.path.join(OUTPUT_DIR, "produse_extrase.xlsx")
        pd.DataFrame(all_rows).to_excel(out_xlsx, index=False)
        print(f"\n💾 Salvate final: {out_xlsx}")

    print("\n=== REZUMAT ===")
    print("Pagini vizitate:", total_pages_visited)
    print("Total produse:", len(all_rows))
    print("Produse cu EAN:", sum(1 for r in all_rows if r.get('EAN')))
    print("Erori produse:", errors)
    print(f"Log detaliat: {os.path.join(OUTPUT_DIR, 'scraper.log')}")


if __name__ == "__main__":
    main()