Appie/products.py

import json
from pyspark import SparkConf
import requests
import pyspark.sql
from pyspark.sql import SparkSession
from pyspark.conf import SparkConf
from pyspark.sql.types import StructType,StructField, StringType, IntegerType,TimestampType
import pyspark.sql.functions as f

from datetime import timedelta,datetime

import pyodbc

class Appie:
    def __init__(self,spark:SparkSession):
        self.mSpark = spark
        self.mToken = ""
        self.mAnonToken = ""
        self.mHost = "api.ah.nl"
        self.mHostPort = 443

        self.mHeader = {"User-Agent":"Appie/8.50","Accept":"*/*","X-Application":"AHWEBSHOP"}
        
        self.mSQLHost = '192.168.100.24'
        self.mDB = 'appie'
        self.mDBUser = 'job'
        self.mDBPassword = '1kb3nj0b'
        self.mDBODBCDriver = 'MariaDB Unicode'
        self.mODBCString = 'DRIVER={'+self.mDBODBCDriver+'};Server='+self.mSQLHost+';Database='+self.mDB+';User='+self.mDBUser+';Password='+self.mDBPassword

        self.setupTables()
        self.getAnonToken()
        self.newToken()

        

    def setupTables(self):
        
        query = """
                CREATE TABLE auth
                USING jdbc
                OPTIONS(
                    driver 'org.mariadb.jdbc.Driver',
                    user 'job',
                    url 'jdbc:mysql://192.168.100.24/appie&permitMysqlScheme',
                    password '1kb3nj0b',
                    dbtable 'auth',
                    database 'appie'
                    )
                """
        self.mSpark.sql("DROP TABLE IF EXISTS auth;")
        self.mSpark.sql(query)

        query = """
                CREATE TABLE products
                USING jdbc
                OPTIONS(
                    driver 'org.mariadb.jdbc.Driver',
                    user 'job',
                    url 'jdbc:mysql://192.168.100.24/appie&permitMysqlScheme',
                    password '1kb3nj0b',
                    dbtable 'products',
                    database 'appie'
                    )
                """
        self.mSpark.sql("DROP TABLE IF EXISTS products;")
        self.mSpark.sql(query)

        

    def getAnonToken(self):
        #Do we have a valid token?
        df = self.mSpark.read.table("auth")
        df = df.where((df.tokenType == "anon_token") & (df.validTill > datetime.now()))
        if(df.count() >0):
            self.mAnonToken = df.head()["token"]
            return df.head()["token"]
        
        tokenJson = self.appiePost("/mobile-auth/v1/auth/token/anonymous","{\"clientId\":\"appie\"}")
        self.mAnonToken = tokenJson["access_token"]

        validTill = datetime.now() + timedelta(seconds=tokenJson["expires_in"])

        schema=StructType([StructField('id', IntegerType(), True),StructField('tokenType', StringType(), True), StructField('token', StringType(), True), StructField('validTill', TimestampType(), True)])
        newDf = self.mSpark.createDataFrame(schema=schema,data=[(0,'anon_token',tokenJson["access_token"],validTill)])
        newDf.write.mode("append").format("jdbc").saveAsTable("auth")
        return self.mAnonToken

    def appiePost(self,url,body):
        header = self.mHeader
        header.update({"Content-type":"application/json"})
        reply = requests.post(url="https://"+self.mHost+url,headers=header,data=body)
        print(reply.content)
        return reply.json()

    
    def newToken(self) -> str:
        df = self.mSpark.read.table("auth")
        tokenDf = df.where((df.tokenType == "access_token") & (df.validTill > datetime.now()))
        tokenDf.show()
        if(tokenDf.count() >0):
            self.mToken = tokenDf.head()["token"]
            return tokenDf.head()["token"]

        refreshDF = df.where((df.tokenType == "refresh_token") & (df.validTill > datetime.now()))
        refreshDF.show()

        refreshToken = refreshDF.head()["token"]

        refeshBody = "{\"clientId\":\"appie-android\",\"refreshToken\":\"" + refreshToken + "\"}"
        print(refeshBody)
        tokenJson = self.appiePost("/mobile-auth/v1/auth/token/refresh",refeshBody)
        self.mToken = tokenJson["access_token"]

        conn = pyodbc.connect(self.mODBCString)
        curr = conn.cursor()
        curr.execute("update auth set token=\'"+tokenJson["refresh_token"]+"\' where tokenType=\'refresh_token\'")
        curr.commit()
        conn.close()

        validTill = datetime.now() + timedelta(seconds=tokenJson["expires_in"])

        schema=StructType([StructField('id', IntegerType(), True),StructField('tokenType', StringType(), True), StructField('token', StringType(), True), StructField('validTill', TimestampType(), True)])
        newDf = self.mSpark.createDataFrame(schema=schema,data=[(0,'access_token',tokenJson["access_token"],validTill)])
        newDf.write.mode("append").format("jdbc").saveAsTable("auth")
        return self.mToken

    def appieGet(self,url:str,token:str) -> requests.Response:
        link= "https://"+self.mHost+url
        header = self.mHeader
        header.update({"Authorization":"Bearer "+token})
        reply = requests.get(url=link,headers=header)

        return reply

    def searchProduct(self,search:str):
        page = 0
        maxPages =1
        while(page <maxPages):
            url = "/mobile-services/product/search/v2?query="+search+"&sortOn=RELEVANCE&size=1000&page="+str(page)
            productsJson = self.appieGet(url,self.getAnonToken())
            maxPages = productsJson.json()["page"]["totalPages"]
            page = page +1
            
            productsDict = productsJson.json()["products"]
            jsonRDD = self.mSpark.sparkContext.parallelize([json.dumps(productsDict)])
            df = self.mSpark.read.json(jsonRDD,multiLine=True).dropDuplicates(["webshopId"])
            print(search+": "+str(df.count()))

            #if 'unitPriceDescription' not in df.columns:
            #    df = df.withColumn('f', f.lit(''))

            oldDF = self.mSpark.read.format("jdbc").table("products")
            oldPricingDf = self.mSpark.read.format("jdbc").table("pricing").drop("pricingDate")

            changes = list(set(oldPricingDf.columns).difference(df.columns))
            pricingDF = df
            for change in changes:
                pricingDF = pricingDF.withColumn(change,f.lit(''))
            pricingDF = pricingDF.select("webshopId","hqId","unitPriceDescription","currentPrice","priceBeforeBonus")

            changes = list(set(oldDF.columns).difference(df.columns))
            newDF = df
            for change in changes:
                newDF = newDF.withColumn(change,f.lit(''))
            newDF = newDF.select("webshopId","hqId","title","salesUnitSize","orderAvailabilityStatus","mainCategory","subCategory","brand","shopType")
            
            #pricingDF.withColumn("pricingDate",f.current_timestamp()).write.mode("overwrite").format("jdbc").option("driver","org.mariadb.jdbc.Driver").option("user","job").option("password","1kb3nj0b").option("dbtable","pricing").option("url","jdbc:mariadb://192.168.100.24").option("database","appie").saveAsTable("pricing")
            
            newPricingDf = pricingDF.alias("new").join(oldPricingDf.alias("old"),['hqId','webshopId'],how='left').filter("(old.currentPrice is NULL and old.unitPriceDescription is NULL and old.priceBeforeBonus is NULL) or old.unitPriceDescription != new.unitPriceDescription or old.priceBeforeBonus !=  new.priceBeforeBonus or old.currentPrice != new.currentPrice").select('hqId', 'webshopId', 'new.unitPriceDescription','new.currentPrice','new.priceBeforeBonus')
            newPricingDf = newPricingDf.withColumn("pricingDate",f.current_timestamp())
            #newPricingDf.show()
            #.format("jdbc").option("driver","org.mariadb.jdbc.Driver").option("user","job").option("password","1kb3nj0b").option("dbtable","pricing").option("url","jdbc:mariadb://192.168.100.24").option("database","appie")
            newPricingDf.write.mode("append").format("jdbc").option("driver","org.mariadb.jdbc.Driver").option("user","job").option("password","1kb3nj0b").option("dbtable","pricing").option("url","jdbc:mariadb://192.168.100.24").option("database","appie").saveAsTable("pricing")

            cols = ["title","salesUnitSize","orderAvailabilityStatus","mainCategory","subCategory","brand","shopType"]
            

            insertDF = newDF.alias("new").join(oldDF.alias("old"),['hqId','webshopId'],how='left').filter("old.title is NULL").select('new.webshopId','new.hqId',  
                    *(f.coalesce('new.' + col, 'old.' + col).alias(col) for col in cols)
                    )
            #insertDF.show()
            #newDF.write.mode("overwrite").option("driver","org.mariadb.jdbc.Driver").option("user","job").option("password","1kb3nj0b").option("dbtable","products").option("url","jdbc:mariadb://192.168.100.24").option("database","appie").saveAsTable("products")
            insertDF.write.mode("append").format("jdbc").saveAsTable("products")

    def getReceipts(self):
        print(self.mToken)
        print(self.appieGet("/mobile-services/v1/receipts",self.mToken).content)

def init_spark():
    spark = SparkSession.builder.appName("Appie").enableHiveSupport().getOrCreate()
    sc = spark.sparkContext
    return spark,sc

def main():
    spark,sc = init_spark()
    apieInstance = Appie(spark)

    apieInstance.getReceipts()

    apieInstance.searchProduct("brood")
    apieInstance.searchProduct("kaas")
    apieInstance.searchProduct("zeep")
    apieInstance.searchProduct("koek")
    apieInstance.searchProduct("chips")
    apieInstance.searchProduct("groente")
    apieInstance.searchProduct("fruit")
    apieInstance.searchProduct("sap")
    apieInstance.searchProduct("salade")
    apieInstance.searchProduct("pizza")
    apieInstance.searchProduct("hagelslag")
    apieInstance.searchProduct("wasmiddel")
    apieInstance.searchProduct("kruiden")
    apieInstance.searchProduct("ijs")
    apieInstance.searchProduct("maandverband")
    apieInstance.searchProduct("tonijn")
    apieInstance.searchProduct("kat")
    apieInstance.searchProduct("chocolade")
    apieInstance.searchProduct("olie")
    

if __name__ == '__main__':
    main()