diff --git a/README.md b/README.md index 833364e..60339cc 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,161 @@ -# BookStore-PJ2 - -当代数据库管理系统课程实验二 \ No newline at end of file +# bookstore +[![Build Status](https://travis-ci.com/DaSE-DBMS/bookstore.svg?branch=master)](https://travis-ci.com/DaSE-DBMS/bookstore) +[![codecov](https://codecov.io/gh/DaSE-DBMS/bookstore/branch/master/graph/badge.svg)](https://codecov.io/gh/DaSE-DBMS/bookstore) + + +## 功能 + +实现一个提供网上购书功能的网站后端。
+网站支持书商在上面开商店，购买者可以通过网站购买。
+买家和卖家都可以注册自己的账号。
+一个卖家可以开一个或多个网上商店， +买家可以为自已的账户充值，在任意商店购买图书。
+支持 下单->付款->发货->收货 流程。
+ +1.实现对应接口的功能，见项目的doc文件夹下面的.md文件描述 （60%）
+ +其中包括： + +1)用户权限接口，如注册、登录、登出、注销
+ +2)买家用户接口，如充值、下单、付款
+ +3)卖家用户接口，如创建店铺、填加书籍信息及描述、增加库存
+ +通过对应的功能测试，所有test case都pass
+ + +2.为项目添加其它功能 ：（40%）
+ +1)实现后续的流程
+发货 -> 收货 + +2)搜索图书
+用户可以通过关键字搜索，参数化的搜索方式； +如搜索范围包括，题目，标签，目录，内容；全站搜索或是当前店铺搜索。 +如果显示结果较大，需要分页 +(使用全文索引优化查找) + +3)订单状态，订单查询和取消定单
+用户可以查自已的历史订单，用户也可以取消订单。
+取消定单可由买家主动地取消定单，或者买家下单后，经过一段时间超时仍未付款，定单也会自动取消。
+ + +## bookstore目录结构 +``` +bookstore + |-- be 后端 + |-- model 后端逻辑代码 + |-- view 访问后端接口 + |-- .... + |-- doc JSON API规范说明 + |-- fe 前端访问与测试代码 + |-- access + |-- bench 效率测试 + |-- data + |-- book.db sqlite 数据库(book.db，较少量的测试数据) + |-- book_lx.db sqlite 数据库(book_lx.db， 较大量的测试数据，要从网盘下载) + |-- scraper.py 从豆瓣爬取的图书信息数据的代码 + |-- test 功能性测试（包含对前60%功能的测试，不要修改已有的文件，可以提pull request或bug） + |-- conf.py 测试参数，修改这个文件以适应自己的需要 + |-- conftest.py pytest初始化配置，修改这个文件以适应自己的需要 + |-- .... + |-- .... +``` + + +## 安装配置 +安装python (需要python3.6以上) + +进入bookstore文件夹下： + +安装依赖 + + pip install -r requirements.txt + +执行测试 + + bash script/test.sh + +bookstore/fe/data/book.db中包含测试的数据，从豆瓣网抓取的图书信息，其DDL为： + + create table book + ( + id TEXT primary key, + title TEXT, + author TEXT, + publisher TEXT, + original_title TEXT, + translator TEXT, + pub_year TEXT, + pages INTEGER, + price INTEGER, + currency_unit TEXT, + binding TEXT, + isbn TEXT, + author_intro TEXT, + book_intro text, + content TEXT, + tags TEXT, + picture BLOB + ); + + +## 要求 + +3人一组，做好分工，完成下述内容： + +1.bookstore文件夹是该项目的demo，采用flask后端框架与sqlite数据库，实现了前60%功能以及对应的测试用例代码。要求利用ORM使用postgreSQL数据库实现前60%功能，可以在demo的基础上进行修改，也可以采用其他后端框架重新实现。需要通过fe/test/下已有的全部测试用例。 + +2.在完成前60%功能的基础上，继续实现后40%功能，要有接口、后端逻辑实现、数据库操作、代码测试。对所有接口都要写test case，通过测试并计算测试覆盖率（尽量提高测试覆盖率）。 + +3.尽量使用索引、事务处理等关系数据库特性，对程序与数据库执行的性能有考量 + +4.尽量使用git等版本管理工具 + +5.不需要实现界面，通过代码测试体现功能与正确性 + + +## 报告内容 + +1.每位组员的学号、姓名，以及分工 + +2.关系数据库设计：概念设计、ER图、关系模式等 + +3.对60%基础功能和40%附加功能的接口、后端逻辑、数据库操作、测试用例进行介绍，展示测试结果与测试覆盖率。 + +4.如果完成，可以展示本次大作业的亮点，比如要求中的“3 4”两点。 + +注：验收依据为报告，本次大作业所作的工作要完整展示在报告中。 + + +## 验收与考核准测 + +- 提交 **代码+报告** 压缩包到 **第二次大作业提交** 入口，命名规则：2022_CDMS_PJ2_第几组 +- 提交截止日期：**2022.12.10 22:00** + +本次大作业不需要提交演示视频，验收的依据是报告： + +1. 没有提交或没有实质的工作，得D +2. 完成"要求"中的第1点，可得C +3. 完成前2点，通过全部测试用例且有较高的测试覆盖率，可得B +4. 完成前2点的基础上，体现出第3 4点，可得A + + +## 附加任务 + +本次考核不做要求 + +学有余力的同学可以尝试下述内容，可以写在报告里： + +更多的数据 book_lx.db 可以从网盘下载，下载地址为： + + https://pan.baidu.com/s/1bjCOW8Z5N_ClcqU54Pdt8g + +提取码： + + hj6q + +这份数据同bookstore/fe/data/book.db的schema相同，但是有更多的数据(约3.5GB, 40000+行) + +可以将book_lx.db导入到数据库中，测试下单及付款两个接口的性能（最好分离负载生成和后端），测出支持的每分钟交易数，延迟等。 diff --git a/be/__init__.py b/be/__init__.py new file mode 100644 index 0000000..e5a0d9b --- /dev/null +++ b/be/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python3 diff --git a/be/app.py b/be/app.py new file mode 100644 index 0000000..e0a6207 --- /dev/null +++ b/be/app.py @@ -0,0 +1,4 @@ +from be import serve + +if __name__ == "__main__": + serve.be_run() diff --git a/be/model/__init__.py b/be/model/__init__.py new file mode 100644 index 0000000..e5a0d9b --- /dev/null +++ b/be/model/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python3 diff --git a/be/model/buyer.py b/be/model/buyer.py new file mode 100644 index 0000000..f8b6595 --- /dev/null +++ b/be/model/buyer.py @@ -0,0 +1,162 @@ +import sqlite3 as sqlite +import uuid +import json +import logging +from be.model import db_conn +from be.model import error + + +class Buyer(db_conn.DBConn): + def __init__(self): + db_conn.DBConn.__init__(self) + + def new_order(self, user_id: str, store_id: str, id_and_count: [(str, int)]) -> (int, str, str): + order_id = "" + try: + if not self.user_id_exist(user_id): + return error.error_non_exist_user_id(user_id) + (order_id, ) + if not self.store_id_exist(store_id): + return error.error_non_exist_store_id(store_id) + (order_id, ) + uid = "{}_{}_{}".format(user_id, store_id, str(uuid.uuid1())) + + for book_id, count in id_and_count: + cursor = self.conn.execute( + "SELECT book_id, stock_level, book_info FROM store " + "WHERE store_id = ? AND book_id = ?;", + (store_id, book_id)) + row = cursor.fetchone() + if row is None: + return error.error_non_exist_book_id(book_id) + (order_id, ) + + stock_level = row[1] + book_info = row[2] + book_info_json = json.loads(book_info) + price = book_info_json.get("price") + + if stock_level < count: + return error.error_stock_level_low(book_id) + (order_id,) + + cursor = self.conn.execute( + "UPDATE store set stock_level = stock_level - ? " + "WHERE store_id = ? and book_id = ? and stock_level >= ?; ", + (count, store_id, book_id, count)) + if cursor.rowcount == 0: + return error.error_stock_level_low(book_id) + (order_id, ) + + self.conn.execute( + "INSERT INTO new_order_detail(order_id, book_id, count, price) " + "VALUES(?, ?, ?, ?);", + (uid, book_id, count, price)) + + self.conn.execute( + "INSERT INTO new_order(order_id, store_id, user_id) " + "VALUES(?, ?, ?);", + (uid, store_id, user_id)) + self.conn.commit() + order_id = uid + except sqlite.Error as e: + logging.info("528, {}".format(str(e))) + return 528, "{}".format(str(e)), "" + except BaseException as e: + logging.info("530, {}".format(str(e))) + return 530, "{}".format(str(e)), "" + + return 200, "ok", order_id + + def payment(self, user_id: str, password: str, order_id: str) -> (int, str): + conn = self.conn + try: + cursor = conn.execute("SELECT order_id, user_id, store_id FROM new_order WHERE order_id = ?", (order_id,)) + row = cursor.fetchone() + if row is None: + return error.error_invalid_order_id(order_id) + + order_id = row[0] + buyer_id = row[1] + store_id = row[2] + + if buyer_id != user_id: + return error.error_authorization_fail() + + cursor = conn.execute("SELECT balance, password FROM user WHERE user_id = ?;", (buyer_id,)) + row = cursor.fetchone() + if row is None: + return error.error_non_exist_user_id(buyer_id) + balance = row[0] + if password != row[1]: + return error.error_authorization_fail() + + cursor = conn.execute("SELECT store_id, user_id FROM user_store WHERE store_id = ?;", (store_id,)) + row = cursor.fetchone() + if row is None: + return error.error_non_exist_store_id(store_id) + + seller_id = row[1] + + if not self.user_id_exist(seller_id): + return error.error_non_exist_user_id(seller_id) + + cursor = conn.execute("SELECT book_id, count, price FROM new_order_detail WHERE order_id = ?;", (order_id,)) + total_price = 0 + for row in cursor: + count = row[1] + price = row[2] + total_price = total_price + price * count + + if balance < total_price: + return error.error_not_sufficient_funds(order_id) + + cursor = conn.execute("UPDATE user set balance = balance - ?" + "WHERE user_id = ? AND balance >= ?", + (total_price, buyer_id, total_price)) + if cursor.rowcount == 0: + return error.error_not_sufficient_funds(order_id) + + cursor = conn.execute("UPDATE user set balance = balance + ?" + "WHERE user_id = ?", + (total_price, buyer_id)) + + if cursor.rowcount == 0: + return error.error_non_exist_user_id(buyer_id) + + cursor = conn.execute("DELETE FROM new_order WHERE order_id = ?", (order_id, )) + if cursor.rowcount == 0: + return error.error_invalid_order_id(order_id) + + cursor = conn.execute("DELETE FROM new_order_detail where order_id = ?", (order_id, )) + if cursor.rowcount == 0: + return error.error_invalid_order_id(order_id) + + conn.commit() + + except sqlite.Error as e: + return 528, "{}".format(str(e)) + + except BaseException as e: + return 530, "{}".format(str(e)) + + return 200, "ok" + + def add_funds(self, user_id, password, add_value) -> (int, str): + try: + cursor = self.conn.execute("SELECT password from user where user_id=?", (user_id,)) + row = cursor.fetchone() + if row is None: + return error.error_authorization_fail() + + if row[0] != password: + return error.error_authorization_fail() + + cursor = self.conn.execute( + "UPDATE user SET balance = balance + ? WHERE user_id = ?", + (add_value, user_id)) + if cursor.rowcount == 0: + return error.error_non_exist_user_id(user_id) + + self.conn.commit() + except sqlite.Error as e: + return 528, "{}".format(str(e)) + except BaseException as e: + return 530, "{}".format(str(e)) + + return 200, "ok" diff --git a/be/model/db_conn.py b/be/model/db_conn.py new file mode 100644 index 0000000..f322cfc --- /dev/null +++ b/be/model/db_conn.py @@ -0,0 +1,30 @@ +from be.model import store + + +class DBConn: + def __init__(self): + self.conn = store.get_db_conn() + + def user_id_exist(self, user_id): + cursor = self.conn.execute("SELECT user_id FROM user WHERE user_id = ?;", (user_id,)) + row = cursor.fetchone() + if row is None: + return False + else: + return True + + def book_id_exist(self, store_id, book_id): + cursor = self.conn.execute("SELECT book_id FROM store WHERE store_id = ? AND book_id = ?;", (store_id, book_id)) + row = cursor.fetchone() + if row is None: + return False + else: + return True + + def store_id_exist(self, store_id): + cursor = self.conn.execute("SELECT store_id FROM user_store WHERE store_id = ?;", (store_id,)) + row = cursor.fetchone() + if row is None: + return False + else: + return True diff --git a/be/model/error.py b/be/model/error.py new file mode 100644 index 0000000..0b25f2a --- /dev/null +++ b/be/model/error.py @@ -0,0 +1,66 @@ + +error_code = { + 401: "authorization fail.", + 511: "non exist user id {}", + 512: "exist user id {}", + 513: "non exist store id {}", + 514: "exist store id {}", + 515: "non exist book id {}", + 516: "exist book id {}", + 517: "stock level low, book id {}", + 518: "invalid order id {}", + 519: "not sufficient funds, order id {}", + 520: "", + 521: "", + 522: "", + 523: "", + 524: "", + 525: "", + 526: "", + 527: "", + 528: "", +} + + +def error_non_exist_user_id(user_id): + return 511, error_code[511].format(user_id) + + +def error_exist_user_id(user_id): + return 512, error_code[512].format(user_id) + + +def error_non_exist_store_id(store_id): + return 513, error_code[513].format(store_id) + + +def error_exist_store_id(store_id): + return 514, error_code[514].format(store_id) + + +def error_non_exist_book_id(book_id): + return 515, error_code[515].format(book_id) + + +def error_exist_book_id(book_id): + return 516, error_code[516].format(book_id) + + +def error_stock_level_low(book_id): + return 517, error_code[517].format(book_id) + + +def error_invalid_order_id(order_id): + return 518, error_code[518].format(order_id) + + +def error_not_sufficient_funds(order_id): + return 519, error_code[518].format(order_id) + + +def error_authorization_fail(): + return 401, error_code[401] + + +def error_and_message(code, message): + return code, message diff --git a/be/model/seller.py b/be/model/seller.py new file mode 100644 index 0000000..2cf47aa --- /dev/null +++ b/be/model/seller.py @@ -0,0 +1,60 @@ +import sqlite3 as sqlite +from be.model import error +from be.model import db_conn + + +class Seller(db_conn.DBConn): + + def __init__(self): + db_conn.DBConn.__init__(self) + + def add_book(self, user_id: str, store_id: str, book_id: str, book_json_str: str, stock_level: int): + try: + if not self.user_id_exist(user_id): + return error.error_non_exist_user_id(user_id) + if not self.store_id_exist(store_id): + return error.error_non_exist_store_id(store_id) + if self.book_id_exist(store_id, book_id): + return error.error_exist_book_id(book_id) + + self.conn.execute("INSERT into store(store_id, book_id, book_info, stock_level)" + "VALUES (?, ?, ?, ?)", (store_id, book_id, book_json_str, stock_level)) + self.conn.commit() + except sqlite.Error as e: + return 528, "{}".format(str(e)) + except BaseException as e: + return 530, "{}".format(str(e)) + return 200, "ok" + + def add_stock_level(self, user_id: str, store_id: str, book_id: str, add_stock_level: int): + try: + if not self.user_id_exist(user_id): + return error.error_non_exist_user_id(user_id) + if not self.store_id_exist(store_id): + return error.error_non_exist_store_id(store_id) + if not self.book_id_exist(store_id, book_id): + return error.error_non_exist_book_id(book_id) + + self.conn.execute("UPDATE store SET stock_level = stock_level + ? " + "WHERE store_id = ? AND book_id = ?", (add_stock_level, store_id, book_id)) + self.conn.commit() + except sqlite.Error as e: + return 528, "{}".format(str(e)) + except BaseException as e: + return 530, "{}".format(str(e)) + return 200, "ok" + + def create_store(self, user_id: str, store_id: str) -> (int, str): + try: + if not self.user_id_exist(user_id): + return error.error_non_exist_user_id(user_id) + if self.store_id_exist(store_id): + return error.error_exist_store_id(store_id) + self.conn.execute("INSERT into user_store(store_id, user_id)" + "VALUES (?, ?)", (store_id, user_id)) + self.conn.commit() + except sqlite.Error as e: + return 528, "{}".format(str(e)) + except BaseException as e: + return 530, "{}".format(str(e)) + return 200, "ok" diff --git a/be/model/store.py b/be/model/store.py new file mode 100644 index 0000000..d82aeeb --- /dev/null +++ b/be/model/store.py @@ -0,0 +1,63 @@ +import logging +import os +import sqlite3 as sqlite + + +class Store: + database: str + + def __init__(self, db_path): + self.database = os.path.join(db_path, "be.db") + self.init_tables() + + def init_tables(self): + try: + conn = self.get_db_conn() + conn.execute( + "CREATE TABLE IF NOT EXISTS user (" + "user_id TEXT PRIMARY KEY, password TEXT NOT NULL, " + "balance INTEGER NOT NULL, token TEXT, terminal TEXT);" + ) + + conn.execute( + "CREATE TABLE IF NOT EXISTS user_store(" + "user_id TEXT, store_id, PRIMARY KEY(user_id, store_id));" + ) + + conn.execute( + "CREATE TABLE IF NOT EXISTS store( " + "store_id TEXT, book_id TEXT, book_info TEXT, stock_level INTEGER," + " PRIMARY KEY(store_id, book_id))" + ) + + conn.execute( + "CREATE TABLE IF NOT EXISTS new_order( " + "order_id TEXT PRIMARY KEY, user_id TEXT, store_id TEXT)" + ) + + conn.execute( + "CREATE TABLE IF NOT EXISTS new_order_detail( " + "order_id TEXT, book_id TEXT, count INTEGER, price INTEGER, " + "PRIMARY KEY(order_id, book_id))" + ) + + conn.commit() + except sqlite.Error as e: + logging.error(e) + conn.rollback() + + def get_db_conn(self) -> sqlite.Connection: + return sqlite.connect(self.database) + + +database_instance: Store = None + + +def init_database(db_path): + global database_instance + database_instance = Store(db_path) + + +def get_db_conn(): + global database_instance + return database_instance.get_db_conn() diff --git a/be/model/user.py b/be/model/user.py new file mode 100644 index 0000000..acd58c2 --- /dev/null +++ b/be/model/user.py @@ -0,0 +1,169 @@ +import jwt +import time +import logging +import sqlite3 as sqlite +from be.model import error +from be.model import db_conn + +# encode a json string like: +# { +# "user_id": [user name], +# "terminal": [terminal code], +# "timestamp": [ts]} to a JWT +# } + + +def jwt_encode(user_id: str, terminal: str) -> str: + encoded = jwt.encode( + {"user_id": user_id, "terminal": terminal, "timestamp": time.time()}, + key=user_id, + algorithm="HS256", + ) + return encoded.encode("utf-8").decode("utf-8") + + +# decode a JWT to a json string like: +# { +# "user_id": [user name], +# "terminal": [terminal code], +# "timestamp": [ts]} to a JWT +# } +def jwt_decode(encoded_token, user_id: str) -> str: + decoded = jwt.decode(encoded_token, key=user_id, algorithms="HS256") + return decoded + + +class User(db_conn.DBConn): + token_lifetime: int = 3600 # 3600 second + + def __init__(self): + db_conn.DBConn.__init__(self) + + def __check_token(self, user_id, db_token, token) -> bool: + try: + if db_token != token: + return False + jwt_text = jwt_decode(encoded_token=token, user_id=user_id) + ts = jwt_text["timestamp"] + if ts is not None: + now = time.time() + if self.token_lifetime > now - ts >= 0: + return True + except jwt.exceptions.InvalidSignatureError as e: + logging.error(str(e)) + return False + + def register(self, user_id: str, password: str): + try: + terminal = "terminal_{}".format(str(time.time())) + token = jwt_encode(user_id, terminal) + self.conn.execute( + "INSERT into user(user_id, password, balance, token, terminal) " + "VALUES (?, ?, ?, ?, ?);", + (user_id, password, 0, token, terminal), ) + self.conn.commit() + except sqlite.Error: + return error.error_exist_user_id(user_id) + return 200, "ok" + + def check_token(self, user_id: str, token: str) -> (int, str): + cursor = self.conn.execute("SELECT token from user where user_id=?", (user_id,)) + row = cursor.fetchone() + if row is None: + return error.error_authorization_fail() + db_token = row[0] + if not self.__check_token(user_id, db_token, token): + return error.error_authorization_fail() + return 200, "ok" + + def check_password(self, user_id: str, password: str) -> (int, str): + cursor = self.conn.execute("SELECT password from user where user_id=?", (user_id,)) + row = cursor.fetchone() + if row is None: + return error.error_authorization_fail() + + if password != row[0]: + return error.error_authorization_fail() + + return 200, "ok" + + def login(self, user_id: str, password: str, terminal: str) -> (int, str, str): + token = "" + try: + code, message = self.check_password(user_id, password) + if code != 200: + return code, message, "" + + token = jwt_encode(user_id, terminal) + cursor = self.conn.execute( + "UPDATE user set token= ? , terminal = ? where user_id = ?", + (token, terminal, user_id), ) + if cursor.rowcount == 0: + return error.error_authorization_fail() + ("", ) + self.conn.commit() + except sqlite.Error as e: + return 528, "{}".format(str(e)), "" + except BaseException as e: + return 530, "{}".format(str(e)), "" + return 200, "ok", token + + def logout(self, user_id: str, token: str) -> bool: + try: + code, message = self.check_token(user_id, token) + if code != 200: + return code, message + + terminal = "terminal_{}".format(str(time.time())) + dummy_token = jwt_encode(user_id, terminal) + + cursor = self.conn.execute( + "UPDATE user SET token = ?, terminal = ? WHERE user_id=?", + (dummy_token, terminal, user_id), ) + if cursor.rowcount == 0: + return error.error_authorization_fail() + + self.conn.commit() + except sqlite.Error as e: + return 528, "{}".format(str(e)) + except BaseException as e: + return 530, "{}".format(str(e)) + return 200, "ok" + + def unregister(self, user_id: str, password: str) -> (int, str): + try: + code, message = self.check_password(user_id, password) + if code != 200: + return code, message + + cursor = self.conn.execute("DELETE from user where user_id=?", (user_id,)) + if cursor.rowcount == 1: + self.conn.commit() + else: + return error.error_authorization_fail() + except sqlite.Error as e: + return 528, "{}".format(str(e)) + except BaseException as e: + return 530, "{}".format(str(e)) + return 200, "ok" + + def change_password(self, user_id: str, old_password: str, new_password: str) -> bool: + try: + code, message = self.check_password(user_id, old_password) + if code != 200: + return code, message + + terminal = "terminal_{}".format(str(time.time())) + token = jwt_encode(user_id, terminal) + cursor = self.conn.execute( + "UPDATE user set password = ?, token= ? , terminal = ? where user_id = ?", + (new_password, token, terminal, user_id), ) + if cursor.rowcount == 0: + return error.error_authorization_fail() + + self.conn.commit() + except sqlite.Error as e: + return 528, "{}".format(str(e)) + except BaseException as e: + return 530, "{}".format(str(e)) + return 200, "ok" + diff --git a/be/serve.py b/be/serve.py new file mode 100644 index 0000000..6499a7c --- /dev/null +++ b/be/serve.py @@ -0,0 +1,46 @@ +import logging +import os +from flask import Flask +from flask import Blueprint +from flask import request +from be.view import auth +from be.view import seller +from be.view import buyer +from be.model.store import init_database + +bp_shutdown = Blueprint("shutdown", __name__) + + +def shutdown_server(): + func = request.environ.get("werkzeug.server.shutdown") + if func is None: + raise RuntimeError("Not running with the Werkzeug Server") + func() + + +@bp_shutdown.route("/shutdown") +def be_shutdown(): + shutdown_server() + return "Server shutting down..." + + +def be_run(): + this_path = os.path.dirname(__file__) + parent_path = os.path.dirname(this_path) + log_file = os.path.join(parent_path, "app.log") + init_database(parent_path) + + logging.basicConfig(filename=log_file, level=logging.ERROR) + handler = logging.StreamHandler() + formatter = logging.Formatter( + "%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s" + ) + handler.setFormatter(formatter) + logging.getLogger().addHandler(handler) + + app = Flask(__name__) + app.register_blueprint(bp_shutdown) + app.register_blueprint(auth.bp_auth) + app.register_blueprint(seller.bp_seller) + app.register_blueprint(buyer.bp_buyer) + app.run() diff --git a/be/view/__init__.py b/be/view/__init__.py new file mode 100644 index 0000000..e5a0d9b --- /dev/null +++ b/be/view/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python3 diff --git a/be/view/auth.py b/be/view/auth.py new file mode 100644 index 0000000..8d9528a --- /dev/null +++ b/be/view/auth.py @@ -0,0 +1,53 @@ +from flask import Blueprint +from flask import request +from flask import jsonify +from be.model import user + +bp_auth = Blueprint("auth", __name__, url_prefix="/auth") + + +@bp_auth.route("/login", methods=["POST"]) +def login(): + user_id = request.json.get("user_id", "") + password = request.json.get("password", "") + terminal = request.json.get("terminal", "") + u = user.User() + code, message, token = u.login(user_id=user_id, password=password, terminal=terminal) + return jsonify({"message": message, "token": token}), code + + +@bp_auth.route("/logout", methods=["POST"]) +def logout(): + user_id: str = request.json.get("user_id") + token: str = request.headers.get("token") + u = user.User() + code, message = u.logout(user_id=user_id, token=token) + return jsonify({"message": message}), code + + +@bp_auth.route("/register", methods=["POST"]) +def register(): + user_id = request.json.get("user_id", "") + password = request.json.get("password", "") + u = user.User() + code, message = u.register(user_id=user_id, password=password) + return jsonify({"message": message}), code + + +@bp_auth.route("/unregister", methods=["POST"]) +def unregister(): + user_id = request.json.get("user_id", "") + password = request.json.get("password", "") + u = user.User() + code, message = u.unregister(user_id=user_id, password=password) + return jsonify({"message": message}), code + + +@bp_auth.route("/password", methods=["POST"]) +def change_password(): + user_id = request.json.get("user_id", "") + old_password = request.json.get("oldPassword", "") + new_password = request.json.get("newPassword", "") + u = user.User() + code, message = u.change_password(user_id=user_id, old_password=old_password, new_password=new_password) + return jsonify({"message": message}), code diff --git a/be/view/buyer.py b/be/view/buyer.py new file mode 100644 index 0000000..ce26221 --- /dev/null +++ b/be/view/buyer.py @@ -0,0 +1,42 @@ +from flask import Blueprint +from flask import request +from flask import jsonify +from be.model.buyer import Buyer + +bp_buyer = Blueprint("buyer", __name__, url_prefix="/buyer") + + +@bp_buyer.route("/new_order", methods=["POST"]) +def new_order(): + user_id: str = request.json.get("user_id") + store_id: str = request.json.get("store_id") + books: [] = request.json.get("books") + id_and_count = [] + for book in books: + book_id = book.get("id") + count = book.get("count") + id_and_count.append((book_id, count)) + + b = Buyer() + code, message, order_id = b.new_order(user_id, store_id, id_and_count) + return jsonify({"message": message, "order_id": order_id}), code + + +@bp_buyer.route("/payment", methods=["POST"]) +def payment(): + user_id: str = request.json.get("user_id") + order_id: str = request.json.get("order_id") + password: str = request.json.get("password") + b = Buyer() + code, message = b.payment(user_id, password, order_id) + return jsonify({"message": message}), code + + +@bp_buyer.route("/add_funds", methods=["POST"]) +def add_funds(): + user_id = request.json.get("user_id") + password = request.json.get("password") + add_value = request.json.get("add_value") + b = Buyer() + code, message = b.add_funds(user_id, password, add_value) + return jsonify({"message": message}), code diff --git a/be/view/seller.py b/be/view/seller.py new file mode 100644 index 0000000..3f56eed --- /dev/null +++ b/be/view/seller.py @@ -0,0 +1,42 @@ +from flask import Blueprint +from flask import request +from flask import jsonify +from be.model import seller +import json + +bp_seller = Blueprint("seller", __name__, url_prefix="/seller") + + +@bp_seller.route("/create_store", methods=["POST"]) +def seller_create_store(): + user_id: str = request.json.get("user_id") + store_id: str = request.json.get("store_id") + s = seller.Seller() + code, message = s.create_store(user_id, store_id) + return jsonify({"message": message}), code + + +@bp_seller.route("/add_book", methods=["POST"]) +def seller_add_book(): + user_id: str = request.json.get("user_id") + store_id: str = request.json.get("store_id") + book_info: str = request.json.get("book_info") + stock_level: str = request.json.get("stock_level", 0) + + s = seller.Seller() + code, message = s.add_book(user_id, store_id, book_info.get("id"), json.dumps(book_info), stock_level) + + return jsonify({"message": message}), code + + +@bp_seller.route("/add_stock_level", methods=["POST"]) +def add_stock_level(): + user_id: str = request.json.get("user_id") + store_id: str = request.json.get("store_id") + book_id: str = request.json.get("book_id") + add_num: str = request.json.get("add_stock_level", 0) + + s = seller.Seller() + code, message = s.add_stock_level(user_id, store_id, book_id, add_num) + + return jsonify({"message": message}), code diff --git a/doc/auth.md b/doc/auth.md new file mode 100644 index 0000000..ba7bc4b --- /dev/null +++ b/doc/auth.md @@ -0,0 +1,213 @@ +## 注册用户 + +#### URL： +POST http://$address$/auth/register + +#### Request + +Body: +``` +{ + "user_id":"$user name$", + "password":"$user password$" +} +``` + +变量名 | 类型 | 描述 | 是否可为空 +---|---|---|--- +user_id | string | 用户名 | N +password | string | 登陆密码 | N + +#### Response + +Status Code: + + +码 | 描述 +--- | --- +200 | 注册成功 +5XX | 注册失败，用户名重复 + +Body: +``` +{ + "message":"$error message$" +} +``` +变量名 | 类型 | 描述 | 是否可为空 +---|---|---|--- +message | string | 返回错误消息，成功时为"ok" | N + +## 注销用户 + +#### URL： +POST http://$address$/auth/unregister + +#### Request + +Body: +``` +{ + "user_id":"$user name$", + "password":"$user password$" +} +``` + +变量名 | 类型 | 描述 | 是否可为空 +---|---|---|--- +user_id | string | 用户名 | N +password | string | 登陆密码 | N + +#### Response + +Status Code: + + +码 | 描述 +--- | --- +200 | 注销成功 +401 | 注销失败，用户名不存在或密码不正确 + + +Body: +``` +{ + "message":"$error message$" +} +``` +变量名 | 类型 | 描述 | 是否可为空 +---|---|---|--- +message | string | 返回错误消息，成功时为"ok" | N + +## 用户登录 + +#### URL： +POST http://$address$/auth/login + +#### Request + +Body: +``` +{ + "user_id":"$user name$", + "password":"$user password$", + "terminal":"$terminal code$" +} +``` + +变量名 | 类型 | 描述 | 是否可为空 +---|---|---|--- +user_id | string | 用户名 | N +password | string | 登陆密码 | N +terminal | string | 终端代码 | N + +#### Response + +Status Code: + +码 | 描述 +--- | --- +200 | 登录成功 +401 | 登录失败，用户名或密码错误 + +Body: +``` +{ + "message":"$error message$", + "token":"$access token$" +} +``` +变量名 | 类型 | 描述 | 是否可为空 +---|---|---|--- +message | string | 返回错误消息，成功时为"ok" | N +token | string | 访问token，用户登录后每个需要授权的请求应在headers中传入这个token | 成功时不为空 + +#### 说明 + +1.terminal标识是哪个设备登录的，不同的设备拥有不同的ID，测试时可以随机生成。 + +2.token是登录后，在客户端中缓存的令牌，在用户登录时由服务端生成，用户在接下来的访问请求时不需要密码。token会定期地失效，对于不同的设备，token是不同的。token只对特定的时期特定的设备是有效的。 + +## 用户更改密码 + +#### URL： +POST http://$address$/auth/password + +#### Request + +Body: +``` +{ + "user_id":"$user name$", + "oldPassword":"$old password$", + "newPassword":"$new password$" +} +``` + +变量名 | 类型 | 描述 | 是否可为空 +---|---|---|--- +user_id | string | 用户名 | N +oldPassword | string | 旧的登陆密码 | N +newPassword | string | 新的登陆密码 | N + +#### Response + +Status Code: + +码 | 描述 +--- | --- +200 | 更改密码成功 +401 | 更改密码失败 + +Body: +``` +{ + "message":"$error message$", +} +``` +变量名 | 类型 | 描述 | 是否可为空 +---|---|---|--- +message | string | 返回错误消息，成功时为"ok" | N + +## 用户登出 + +#### URL： +POST http://$address$/auth/logout + +#### Request + +Headers: + +key | 类型 | 描述 +---|---|--- +token | string | 访问token + +Body: +``` +{ + "user_id":"$user name$" +} +``` + +变量名 | 类型 | 描述 | 是否可为空 +---|---|---|--- +user_id | string | 用户名 | N + +#### Response + +Status Code: + +码 | 描述 +--- | --- +200 | 登出成功 +401 | 登出失败，用户名或token错误 + +Body: +``` +{ + "message":"$error message$" +} +``` +变量名 | 类型 | 描述 | 是否可为空 +---|---|---|--- +message | string | 返回错误消息，成功时为"ok" | N diff --git a/doc/buyer.md b/doc/buyer.md new file mode 100644 index 0000000..5f415d2 --- /dev/null +++ b/doc/buyer.md @@ -0,0 +1,144 @@ +## 买家下单 + +#### URL： +POST http://[address]/buyer/new_order + +#### Request + +##### Header: + +key | 类型 | 描述 | 是否可为空 +---|---|---|--- +token | string | 登录产生的会话标识 | N + +##### Body: +```json +{ + "user_id": "buyer_id", + "store_id": "store_id", + "books": [ + { + "id": "1000067", + "count": 1 + }, + { + "id": "1000134", + "count": 4 + } + ] +} +``` + +##### 属性说明： + +变量名 | 类型 | 描述 | 是否可为空 +---|---|---|--- +user_id | string | 买家用户ID | N +store_id | string | 商铺ID | N +books | class | 书籍购买列表 | N + +books数组： + +变量名 | 类型 | 描述 | 是否可为空 +---|---|---|--- +id | string | 书籍的ID | N +count | string | 购买数量 | N + + +#### Response + +Status Code: + +码 | 描述 +--- | --- +200 | 下单成功 +5XX | 买家用户ID不存在 +5XX | 商铺ID不存在 +5XX | 购买的图书不存在 +5XX | 商品库存不足 + +##### Body: +```json +{ + "order_id": "uuid" +} +``` + +##### 属性说明： + +变量名 | 类型 | 描述 | 是否可为空 +---|---|---|--- +order_id | string | 订单号，只有返回200时才有效 | N + + +## 买家付款 + +#### URL： +POST http://[address]/buyer/payment + +#### Request + +##### Body: +```json +{ + "user_id": "buyer_id", + "order_id": "order_id", + "password": "password" +} +``` + +##### 属性说明： + +变量名 | 类型 | 描述 | 是否可为空 +---|---|---|--- +user_id | string | 买家用户ID | N +order_id | string | 订单ID | N +password | string | 买家用户密码 | N + + +#### Response + +Status Code: + +码 | 描述 +--- | --- +200 | 付款成功 +5XX | 账户余额不足 +5XX | 无效参数 +401 | 授权失败 + + +## 买家充值 + +#### URL： +POST http://[address]/buyer/add_funds + +#### Request + + + +##### Body: +```json +{ + "user_id": "user_id", + "password": "password", + "add_value": 10 +} +``` + +##### 属性说明： + +key | 类型 | 描述 | 是否可为空 +---|---|---|--- +user_id | string | 买家用户ID | N +password | string | 用户密码 | N +add_value | int | 充值金额，以分为单位 | N + + +Status Code: + +码 | 描述 +--- | --- +200 | 充值成功 +401 | 授权失败 +5XX | 无效参数 diff --git a/doc/seller.md b/doc/seller.md new file mode 100644 index 0000000..0513122 --- /dev/null +++ b/doc/seller.md @@ -0,0 +1,178 @@ +## 创建商铺 + + + +#### URL + +POST http://[address]/seller/create_store + +#### Request +Headers: + +key | 类型 | 描述 | 是否可为空 +---|---|---|--- +token | string | 登录产生的会话标识 | N + +Body: + +```json +{ + "user_id": "$seller id$", + "store_id": "$store id$" +} +``` + +key | 类型 | 描述 | 是否可为空 +---|---|---|--- +user_id | string | 卖家用户ID | N +store_id | string | 商铺ID | N + +#### Response + +Status Code: + +码 | 描述 +--- | --- +200 | 创建商铺成功 +5XX | 商铺ID已存在 + + +## 商家添加书籍信息 + +#### URL： +POST http://[address]/seller/add_book + +#### Request +Headers: + +key | 类型 | 描述 | 是否可为空 +---|---|---|--- +token | string | 登录产生的会话标识 | N + +Body: + +```json +{ + "user_id": "$seller user id$", + "store_id": "$store id$", + "book_info": { + "tags": [ + "tags1", + "tags2", + "tags3", + "..." + ], + "pictures": [ + "$Base 64 encoded bytes array1$", + "$Base 64 encoded bytes array2$", + "$Base 64 encoded bytes array3$", + "..." + ], + "id": "$book id$", + "title": "$book title$", + "author": "$book author$", + "publisher": "$book publisher$", + "original_title": "$original title$", + "translator": "translater", + "pub_year": "$pub year$", + "pages": 10, + "price": 10, + "binding": "平装", + "isbn": "$isbn$", + "author_intro": "$author introduction$", + "book_intro": "$book introduction$", + "content": "$chapter1 ...$" + }, + "stock_level": 0 +} + +``` + +属性说明： + +变量名 | 类型 | 描述 | 是否可为空 +---|---|---|--- +user_id | string | 卖家用户ID | N +store_id | string | 商铺ID | N +book_info | class | 书籍信息 | N +stock_level | int | 初始库存，大于等于0 | N + +book_info类： + +变量名 | 类型 | 描述 | 是否可为空 +---|---|---|--- +id | string | 书籍ID | N +title | string | 书籍题目 | N +author | string | 作者 | Y +publisher | string | 出版社 | Y +original_title | string | 原书题目 | Y +translator | string | 译者 | Y +pub_year | string | 出版年月 | Y +pages | int | 页数 | Y +price | int | 价格(以分为单位) | N +binding | string | 装帧，精状/平装 | Y +isbn | string | ISBN号 | Y +author_intro | string | 作者简介 | Y +book_intro | string | 书籍简介 | Y +content | string | 样章试读 | Y +tags | array | 标签 | Y +pictures | array | 照片 | Y + +tags和pictures： + + tags 中每个数组元素都是string类型 + picture 中每个数组元素都是string（base64表示的bytes array）类型 + + +#### Response + +Status Code: + +码 | 描述 +--- | --- +200 | 添加图书信息成功 +5XX | 卖家用户ID不存在 +5XX | 商铺ID不存在 +5XX | 图书ID已存在 + + +## 商家添加书籍库存 + + +#### URL + +POST http://[address]/seller/add_stock_level + +#### Request +Headers: + +key | 类型 | 描述 | 是否可为空 +---|---|---|--- +token | string | 登录产生的会话标识 | N + +Body: + +```json +{ + "user_id": "$seller id$", + "store_id": "$store id$", + "book_id": "$book id$", + "add_stock_level": 10 +} +``` +key | 类型 | 描述 | 是否可为空 +---|---|---|--- +user_id | string | 卖家用户ID | N +store_id | string | 商铺ID | N +book_id | string | 书籍ID | N +add_stock_level | int | 增加的库存量 | N + +#### Response + +Status Code: + +码 | 描述 +--- | :-- +200 | 创建商铺成功 +5XX | 商铺ID不存在 +5XX | 图书ID不存在 diff --git a/fe/__init__.py b/fe/__init__.py new file mode 100644 index 0000000..e5a0d9b --- /dev/null +++ b/fe/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python3 diff --git a/fe/access/__init__.py b/fe/access/__init__.py new file mode 100644 index 0000000..e5a0d9b --- /dev/null +++ b/fe/access/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python3 diff --git a/fe/access/auth.py b/fe/access/auth.py new file mode 100644 index 0000000..04309fb --- /dev/null +++ b/fe/access/auth.py @@ -0,0 +1,49 @@ +import requests +from urllib.parse import urljoin + + +class Auth: + def __init__(self, url_prefix): + self.url_prefix = urljoin(url_prefix, "auth/") + + def login(self, user_id: str, password: str, terminal: str) -> (int, str): + json = {"user_id": user_id, "password": password, "terminal": terminal} + url = urljoin(self.url_prefix, "login") + r = requests.post(url, json=json) + return r.status_code, r.json().get("token") + + def register( + self, + user_id: str, + password: str + ) -> int: + json = { + "user_id": user_id, + "password": password + } + url = urljoin(self.url_prefix, "register") + r = requests.post(url, json=json) + return r.status_code + + def password(self, user_id: str, old_password: str, new_password: str) -> int: + json = { + "user_id": user_id, + "oldPassword": old_password, + "newPassword": new_password, + } + url = urljoin(self.url_prefix, "password") + r = requests.post(url, json=json) + return r.status_code + + def logout(self, user_id: str, token: str) -> int: + json = {"user_id": user_id} + headers = {"token": token} + url = urljoin(self.url_prefix, "logout") + r = requests.post(url, headers=headers, json=json) + return r.status_code + + def unregister(self, user_id: str, password: str) -> int: + json = {"user_id": user_id, "password": password} + url = urljoin(self.url_prefix, "unregister") + r = requests.post(url, json=json) + return r.status_code diff --git a/fe/access/book.py b/fe/access/book.py new file mode 100644 index 0000000..faa64bd --- /dev/null +++ b/fe/access/book.py @@ -0,0 +1,97 @@ +import os +import sqlite3 as sqlite +import random +import base64 +import simplejson as json + + +class Book: + id: str + title: str + author: str + publisher: str + original_title: str + translator: str + pub_year: str + pages: int + price: int + binding: str + isbn: str + author_intro: str + book_intro: str + content: str + tags: [str] + pictures: [bytes] + + def __init__(self): + self.tags = [] + self.pictures = [] + + +class BookDB: + def __init__(self, large: bool = False): + parent_path = os.path.dirname(os.path.dirname(__file__)) + self.db_s = os.path.join(parent_path, "data/book.db") + self.db_l = os.path.join(parent_path, "data/book_lx.db") + if large: + self.book_db = self.db_l + else: + self.book_db = self.db_s + + def get_book_count(self): + conn = sqlite.connect(self.book_db) + cursor = conn.execute( + "SELECT count(id) FROM book") + row = cursor.fetchone() + return row[0] + + def get_book_info(self, start, size) -> [Book]: + books = [] + conn = sqlite.connect(self.book_db) + cursor = conn.execute( + "SELECT id, title, author, " + "publisher, original_title, " + "translator, pub_year, pages, " + "price, currency_unit, binding, " + "isbn, author_intro, book_intro, " + "content, tags, picture FROM book ORDER BY id " + "LIMIT ? OFFSET ?", (size, start)) + for row in cursor: + book = Book() + book.id = row[0] + book.title = row[1] + book.author = row[2] + book.publisher = row[3] + book.original_title = row[4] + book.translator = row[5] + book.pub_year = row[6] + book.pages = row[7] + book.price = row[8] + + book.currency_unit = row[9] + book.binding = row[10] + book.isbn = row[11] + book.author_intro = row[12] + book.book_intro = row[13] + book.content = row[14] + tags = row[15] + + picture = row[16] + + for tag in tags.split("\n"): + if tag.strip() != "": + book.tags.append(tag) + for i in range(0, random.randint(0, 9)): + if picture is not None: + encode_str = base64.b64encode(picture).decode('utf-8') + book.pictures.append(encode_str) + books.append(book) + # print(tags.decode('utf-8')) + + # print(book.tags, len(book.picture)) + # print(book) + # print(tags) + + return books + + diff --git a/fe/access/buyer.py b/fe/access/buyer.py new file mode 100644 index 0000000..507c62a --- /dev/null +++ b/fe/access/buyer.py @@ -0,0 +1,42 @@ +import requests +import simplejson +from urllib.parse import urljoin +from fe.access.auth import Auth + + +class Buyer: + def __init__(self, url_prefix, user_id, password): + self.url_prefix = urljoin(url_prefix, "buyer/") + self.user_id = user_id + self.password = password + self.token = "" + self.terminal = "my terminal" + self.auth = Auth(url_prefix) + code, self.token = self.auth.login(self.user_id, self.password, self.terminal) + assert code == 200 + + def new_order(self, store_id: str, book_id_and_count: [(str, int)]) -> (int, str): + books = [] + for id_count_pair in book_id_and_count: + books.append({"id": id_count_pair[0], "count": id_count_pair[1]}) + json = {"user_id": self.user_id, "store_id": store_id, "books": books} + #print(simplejson.dumps(json)) + url = urljoin(self.url_prefix, "new_order") + headers = {"token": self.token} + r = requests.post(url, headers=headers, json=json) + response_json = r.json() + return r.status_code, response_json.get("order_id") + + def payment(self, order_id: str): + json = {"user_id": self.user_id, "password": self.password, "order_id": order_id} + url = urljoin(self.url_prefix, "payment") + headers = {"token": self.token} + r = requests.post(url, headers=headers, json=json) + return r.status_code + + def add_funds(self, add_value: str) -> int: + json = {"user_id": self.user_id, "password": self.password, "add_value": add_value} + url = urljoin(self.url_prefix, "add_funds") + headers = {"token": self.token} + r = requests.post(url, headers=headers, json=json) + return r.status_code diff --git a/fe/access/new_buyer.py b/fe/access/new_buyer.py new file mode 100644 index 0000000..d67092a --- /dev/null +++ b/fe/access/new_buyer.py @@ -0,0 +1,10 @@ +from fe import conf +from fe.access import buyer, auth + + +def register_new_buyer(user_id, password) -> buyer.Buyer: + a = auth.Auth(conf.URL) + code = a.register(user_id, password) + assert code == 200 + s = buyer.Buyer(conf.URL, user_id, password) + return s diff --git a/fe/access/new_seller.py b/fe/access/new_seller.py new file mode 100644 index 0000000..0782991 --- /dev/null +++ b/fe/access/new_seller.py @@ -0,0 +1,10 @@ +from fe import conf +from fe.access import seller, auth + + +def register_new_seller(user_id, password) -> seller.Seller: + a = auth.Auth(conf.URL) + code = a.register(user_id, password) + assert code == 200 + s = seller.Seller(conf.URL, user_id, password) + return s diff --git a/fe/access/seller.py b/fe/access/seller.py new file mode 100644 index 0000000..6e20ae8 --- /dev/null +++ b/fe/access/seller.py @@ -0,0 +1,52 @@ +import requests +from urllib.parse import urljoin +from fe.access import book +from fe.access.auth import Auth + + +class Seller: + def __init__(self, url_prefix, seller_id: str, password: str): + self.url_prefix = urljoin(url_prefix, "seller/") + self.seller_id = seller_id + self.password = password + self.terminal = "my terminal" + self.auth = Auth(url_prefix) + code, self.token = self.auth.login(self.seller_id, self.password, self.terminal) + assert code == 200 + + def create_store(self, store_id): + json = { + "user_id": self.seller_id, + "store_id": store_id, + } + #print(simplejson.dumps(json)) + url = urljoin(self.url_prefix, "create_store") + headers = {"token": self.token} + r = requests.post(url, headers=headers, json=json) + return r.status_code + + def add_book(self, store_id: str, stock_level: int, book_info: book.Book) -> int: + json = { + "user_id": self.seller_id, + "store_id": store_id, + "book_info": book_info.__dict__, + "stock_level": stock_level + } + #print(simplejson.dumps(json)) + url = urljoin(self.url_prefix, "add_book") + headers = {"token": self.token} + r = requests.post(url, headers=headers, json=json) + return r.status_code + + def add_stock_level(self, seller_id: str, store_id: str, book_id: str, add_stock_num: int) -> int: + json = { + "user_id": seller_id, + "store_id": store_id, + "book_id": book_id, + "add_stock_level": add_stock_num + } + #print(simplejson.dumps(json)) + url = urljoin(self.url_prefix, "add_stock_level") + headers = {"token": self.token} + r = requests.post(url, headers=headers, json=json) + return r.status_code diff --git a/fe/bench/__init__.py b/fe/bench/__init__.py new file mode 100644 index 0000000..e5a0d9b --- /dev/null +++ b/fe/bench/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python3 diff --git a/fe/bench/bench.md b/fe/bench/bench.md new file mode 100644 index 0000000..ab70cd1 --- /dev/null +++ b/fe/bench/bench.md @@ -0,0 +1 @@ +add performance test here diff --git a/fe/bench/run.py b/fe/bench/run.py new file mode 100644 index 0000000..80043db --- /dev/null +++ b/fe/bench/run.py @@ -0,0 +1,22 @@ +from fe.bench.workload import Workload +from fe.bench.session import Session + + +def run_bench(): + wl = Workload() + wl.gen_database() + + sessions = [] + for i in range(0, wl.session): + ss = Session(wl) + sessions.append(ss) + + for ss in sessions: + ss.start() + + for ss in sessions: + ss.join() + + +#if __name__ == "__main__": +# run_bench() \ No newline at end of file diff --git a/fe/bench/session.py b/fe/bench/session.py new file mode 100644 index 0000000..e0ef1df --- /dev/null +++ b/fe/bench/session.py @@ -0,0 +1,53 @@ +from fe.bench.workload import Workload +from fe.bench.workload import NewOrder +from fe.bench.workload import Payment +import time +import threading + + +class Session(threading.Thread): + def __init__(self, wl: Workload): + threading.Thread.__init__(self) + self.workload = wl + self.new_order_request = [] + self.payment_request = [] + self.payment_i = 0 + self.new_order_i = 0 + self.payment_ok = 0 + self.new_order_ok = 0 + self.time_new_order = 0 + self.time_payment = 0 + self.thread = None + self.gen_procedure() + + def gen_procedure(self): + for i in range(0, self.workload.procedure_per_session): + new_order = self.workload.get_new_order() + self.new_order_request.append(new_order) + + def run(self): + self.run_gut() + + def run_gut(self): + for new_order in self.new_order_request: + before = time.time() + ok, order_id = new_order.run() + after = time.time() + self.time_new_order = self.time_new_order + after - before + self.new_order_i = self.new_order_i + 1 + if ok: + self.new_order_ok = self.new_order_ok + 1 + payment = Payment(new_order.buyer, order_id) + self.payment_request.append(payment) + if self.new_order_i % 100 or self.new_order_i == len(self.new_order_request): + self.workload.update_stat(self.new_order_i, self.payment_i, self.new_order_ok, self.payment_ok, + self.time_new_order, self.time_payment) + for payment in self.payment_request: + before = time.time() + ok = payment.run() + after = time.time() + self.time_payment = self.time_payment + after - before + self.payment_i = self.payment_i + 1 + if ok: + self.payment_ok = self.payment_ok + 1 + self.payment_request = [] diff --git a/fe/bench/workload.py b/fe/bench/workload.py new file mode 100644 index 0000000..d7c9e6e --- /dev/null +++ b/fe/bench/workload.py @@ -0,0 +1,166 @@ +import logging +import uuid +import random +import threading +from fe.access import book +from fe.access.new_seller import register_new_seller +from fe.access.new_buyer import register_new_buyer +from fe.access.buyer import Buyer +from fe import conf + + +class NewOrder: + def __init__(self, buyer: Buyer, store_id, book_id_and_count): + self.buyer = buyer + self.store_id = store_id + self.book_id_and_count = book_id_and_count + + def run(self) -> (bool, str): + code, order_id = self.buyer.new_order(self.store_id, self.book_id_and_count) + return code == 200, order_id + + +class Payment: + def __init__(self, buyer:Buyer, order_id): + self.buyer = buyer + self.order_id = order_id + + def run(self) -> bool: + code = self.buyer.payment(self.order_id) + return code == 200 + + +class Workload: + def __init__(self): + self.uuid = str(uuid.uuid1()) + self.book_ids = [] + self.buyer_ids = [] + self.store_ids = [] + self.book_db = book.BookDB(conf.Use_Large_DB) + self.row_count = self.book_db.get_book_count() + + self.book_num_per_store = conf.Book_Num_Per_Store + if self.row_count < self.book_num_per_store: + self.book_num_per_store = self.row_count + self.store_num_per_user = conf.Store_Num_Per_User + self.seller_num = conf.Seller_Num + self.buyer_num = conf.Buyer_Num + self.session = conf.Session + self.stock_level = conf.Default_Stock_Level + self.user_funds = conf.Default_User_Funds + self.batch_size = conf.Data_Batch_Size + self.procedure_per_session = conf.Request_Per_Session + + self.n_new_order = 0 + self.n_payment = 0 + self.n_new_order_ok = 0 + self.n_payment_ok = 0 + self.time_new_order = 0 + self.time_payment = 0 + self.lock = threading.Lock() + # 存储上一次的值，用于两次做差 + self.n_new_order_past = 0 + self.n_payment_past = 0 + self.n_new_order_ok_past = 0 + self.n_payment_ok_past = 0 + + def to_seller_id_and_password(self, no: int) -> (str, str): + return "seller_{}_{}".format(no, self.uuid), "password_seller_{}_{}".format(no, self.uuid) + + def to_buyer_id_and_password(self, no: int) -> (str, str): + return "buyer_{}_{}".format(no, self.uuid), "buyer_seller_{}_{}".format(no, self.uuid) + + def to_store_id(self, seller_no: int, i): + return "store_s_{}_{}_{}".format(seller_no, i, self.uuid) + + def gen_database(self): + logging.info("load data") + for i in range(1, self.seller_num + 1): + user_id, password = self.to_seller_id_and_password(i) + seller = register_new_seller(user_id, password) + for j in range(1, self.store_num_per_user + 1): + store_id = self.to_store_id(i, j) + code = seller.create_store(store_id) + assert code == 200 + self.store_ids.append(store_id) + row_no = 0 + + while row_no < self.book_num_per_store: + books = self.book_db.get_book_info(row_no, self.batch_size) + if len(books) == 0: + break + for bk in books: + code = seller.add_book(store_id, self.stock_level, bk) + assert code == 200 + if i == 1 and j == 1: + self.book_ids.append(bk.id) + row_no = row_no + len(books) + logging.info("seller data loaded.") + for k in range(1, self.buyer_num + 1): + user_id, password = self.to_buyer_id_and_password(k) + buyer = register_new_buyer(user_id, password) + buyer.add_funds(self.user_funds) + self.buyer_ids.append(user_id) + logging.info("buyer data loaded.") + + def get_new_order(self) -> NewOrder: + n = random.randint(1, self.buyer_num) + buyer_id, buyer_password = self.to_buyer_id_and_password(n) + store_no = int(random.uniform(0, len(self.store_ids) - 1)) + store_id = self.store_ids[store_no] + books = random.randint(1, 10) + book_id_and_count = [] + book_temp = [] + for i in range(0, books): + book_no = int(random.uniform(0, len(self.book_ids) - 1)) + book_id = self.book_ids[book_no] + if book_id in book_temp: + continue + else: + book_temp.append(book_id) + count = random.randint(1, 10) + book_id_and_count.append((book_id, count)) + b = Buyer(url_prefix=conf.URL, user_id=buyer_id, password=buyer_password) + new_ord = NewOrder(b, store_id, book_id_and_count) + return new_ord + + def update_stat(self, n_new_order, n_payment, + n_new_order_ok, n_payment_ok, + time_new_order, time_payment): + # 获取当前并发数 + thread_num = len(threading.enumerate()) + # 加索 + self.lock.acquire() + self.n_new_order = self.n_new_order + n_new_order + self.n_payment = self.n_payment + n_payment + self.n_new_order_ok = self.n_new_order_ok + n_new_order_ok + self.n_payment_ok = self.n_payment_ok + n_payment_ok + self.time_new_order = self.time_new_order + time_new_order + self.time_payment = self.time_payment + time_payment + # 计算这段时间内新创建订单的总数目 + n_new_order_diff = self.n_new_order - self.n_new_order_past + # 计算这段时间内新付款订单的总数目 + n_payment_diff = self.n_payment - self.n_payment_past + + if self.n_payment != 0 and self. n_new_order != 0 \ + and (self.time_payment + self.time_new_order): + # TPS_C(吞吐量):成功创建订单数量/(提交订单时间/提交订单并发数 + 提交付款订单时间/提交付款订单并发数) + # NO=OK:新创建订单数量 + # Thread_num:以新提交订单的数量作为并发数(这一次的TOTAL-上一次的TOTAL) + # TOTAL:总提交订单数量 + # LATENCY:提交订单时间/处理订单笔数(只考虑该线程延迟，未考虑并发) + # P=OK:新创建付款订单数量 + # Thread_num:以新提交付款订单的数量作为并发数(这一次的TOTAL-上一次的TOTAL) + # TOTAL:总付款提交订单数量 + # LATENCY:提交付款订单时间/处理付款订单笔数(只考虑该线程延迟，未考虑并发) + logging.info("TPS_C={}, NO=OK:{} Thread_num:{} TOTAL:{} LATENCY:{} , P=OK:{} Thread_num:{} TOTAL:{} LATENCY:{}".format( + int(self.n_new_order_ok / (self.time_payment / n_payment_diff + self.time_new_order / n_new_order_diff)), # 吞吐量:完成订单数/((付款所用时间+订单所用时间)/并发数) + self.n_new_order_ok, n_new_order_diff, self.n_new_order, self.time_new_order / self.n_new_order, # 订单延迟:(创建订单所用时间/并发数)/新创建订单数 + self.n_payment_ok, n_payment_diff, self.n_payment, self.time_payment / self.n_payment # 付款延迟:(付款所用时间/并发数)/付款订单数 + )) + self.lock.release() + # 旧值更新为新值，便于下一轮计算 + self.n_new_order_past = self.n_new_order + self.n_payment_past = self.n_payment + self.n_new_order_ok_past = self.n_new_order_ok + self.n_payment_ok_past = self.n_payment_ok diff --git a/fe/conf.py b/fe/conf.py new file mode 100644 index 0000000..a0fbb2b --- /dev/null +++ b/fe/conf.py @@ -0,0 +1,11 @@ +URL = "http://127.0.0.1:5000/" +Book_Num_Per_Store = 2000 +Store_Num_Per_User = 2 +Seller_Num = 2 +Buyer_Num = 10 +Session = 1 +Request_Per_Session = 1000 +Default_Stock_Level = 1000000 +Default_User_Funds = 10000000 +Data_Batch_Size = 100 +Use_Large_DB = False diff --git a/fe/conftest.py b/fe/conftest.py new file mode 100644 index 0000000..e93cee2 --- /dev/null +++ b/fe/conftest.py @@ -0,0 +1,27 @@ +import requests +import threading +from urllib.parse import urljoin +from be import serve +from fe import conf + +thread: threading.Thread = None + + +# 修改这里启动后端程序，如果不需要可删除这行代码 +def run_backend(): + # rewrite this if rewrite backend + serve.be_run() + + +def pytest_configure(config): + global thread + print("frontend begin test") + thread = threading.Thread(target=run_backend) + thread.start() + + +def pytest_unconfigure(config): + url = urljoin(conf.URL, "shutdown") + requests.get(url) + thread.join() + print("frontend end test") diff --git a/fe/data/book.db b/fe/data/book.db new file mode 100644 index 0000000..5ee601c Binary files /dev/null and b/fe/data/book.db differ diff --git a/fe/data/scraper.py b/fe/data/scraper.py new file mode 100644 index 0000000..c85e3d1 --- /dev/null +++ b/fe/data/scraper.py @@ -0,0 +1,425 @@ +# coding=utf-8 + +from lxml import etree +import sqlite3 +import re +import requests +import random +import time +import logging + +user_agent = [ + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 " + "Safari/534.50", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 " + "Safari/534.50", + "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0", + "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR " + "3.0.30729; .NET CLR 3.5.30729; InfoPath.3; rv:11.0) like Gecko", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)", + "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)", + "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)", + "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1", + "Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1", + "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11", + "Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 " + "Safari/535.11", + "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)", + "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)", + "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)", + "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)", + "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET " + "CLR 2.0.50727; SE 2.X MetaSr 1.0)", + "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)", + "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser)", + "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)", + "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) " + "Version/5.0.2 Mobile/8J2 Safari/6533.18.5", + "Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) " + "Version/5.0.2 Mobile/8J2 Safari/6533.18.5", + "Mozilla/5.0 (iPad; U; CPU OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) " + "Version/5.0.2 Mobile/8J2 Safari/6533.18.5", + "Mozilla/5.0 (Linux; U; Android 2.3.7; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) " + "Version/4.0 Mobile Safari/533.1", + "MQQBrowser/26 Mozilla/5.0 (Linux; U; Android 2.3.7; zh-cn; MB200 Build/GRJ22; CyanogenMod-7) " + "AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1", + "Opera/9.80 (Android 2.3.4; Linux; Opera Mobi/build-1107180945; U; en-GB) Presto/2.8.149 Version/11.10", + "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/534.13 (KHTML, like Gecko) " + "Version/4.0 Safari/534.13", + "Mozilla/5.0 (BlackBerry; U; BlackBerry 9800; en) AppleWebKit/534.1+ (KHTML, like Gecko) Version/6.0.0.337 " + "Mobile Safari/534.1+", + "Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.0; U; en-US) AppleWebKit/534.6 (KHTML, like Gecko) " + "wOSBrowser/233.70 Safari/534.6 TouchPad/1.0", + "Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019; Profile/MIDP-2.1 Configuration/CLDC-1.1) " + "AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.18124", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0; HTC; Titan)", + "UCWEB7.0.2.37/28/999", + "NOKIA5700/ UCWEB7.0.2.37/28/999", + "Openwave/ UCWEB7.0.2.37/28/999", + "Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/999", + # iPhone 6： + "Mozilla/6.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/8.0 " + "Mobile/10A5376e Safari/8536.25", +] + + +def get_user_agent(): + headers = {"User-Agent": random.choice(user_agent)} + return headers + + +class Scraper: + database: str + tag: str + page: int + + def __init__(self): + self.database = "book.db" + self.tag = "" + self.page = 0 + self.pattern_number = re.compile(r"\d+\.?\d*") + logging.basicConfig(filename="scraper.log", level=logging.ERROR) + + def get_current_progress(self) -> (): + conn = sqlite3.connect(self.database) + results = conn.execute("SELECT tag, page from progress where id = '0'") + for row in results: + return row[0], row[1] + return "", 0 + + def save_current_progress(self, current_tag, current_page): + conn = sqlite3.connect(self.database) + conn.execute( + "UPDATE progress set tag = '{}', page = {} where id = '0'".format( + current_tag, current_page + ) + ) + conn.commit() + conn.close() + + def start_grab(self) -> bool: + self.create_tables() + scraper.grab_tag() + current_tag, current_page = self.get_current_progress() + tags = self.get_tag_list() + for i in range(0, len(tags)): + no = 0 + if i == 0 and current_tag == tags[i]: + no = current_page + while self.grab_book_list(tags[i], no): + no = no + 20 + return True + + def create_tables(self): + conn = sqlite3.connect(self.database) + try: + conn.execute("CREATE TABLE tags (tag TEXT PRIMARY KEY)") + conn.commit() + except sqlite3.Error as e: + logging.error(str(e)) + conn.rollback() + + try: + conn.execute( + "CREATE TABLE book (" + "id TEXT PRIMARY KEY, title TEXT, author TEXT, " + "publisher TEXT, original_title TEXT, " + "translator TEXT, pub_year TEXT, pages INTEGER, " + "price INTEGER, currency_unit TEXT, binding TEXT, " + "isbn TEXT, author_intro TEXT, book_intro text, " + "content TEXT, tags TEXT, picture BLOB)" + ) + conn.commit() + except sqlite3.Error as e: + logging.error(str(e)) + conn.rollback() + + try: + conn.execute( + "CREATE TABLE progress (id TEXT PRIMARY KEY, tag TEXT, page integer )" + ) + conn.execute("INSERT INTO progress values('0', '', 0)") + conn.commit() + except sqlite3.Error as e: + logging.error(str(e)) + conn.rollback() + + def grab_tag(self): + url = "https://book.douban.com/tag/?view=cloud" + r = requests.get(url, headers=get_user_agent()) + r.encoding = "utf-8" + h: etree.ElementBase = etree.HTML(r.text) + tags: [] = h.xpath( + '/html/body/div[@id="wrapper"]/div[@id="content"]' + '/div[@class="grid-16-8 clearfix"]/div[@class="article"]' + '/div[@class=""]/div[@class="indent tag_cloud"]' + "/table/tbody/tr/td/a/@href" + ) + conn = sqlite3.connect(self.database) + c = conn.cursor() + try: + for tag in tags: + t: str = tag.strip("/tag") + c.execute("INSERT INTO tags VALUES ('{}')".format(t)) + c.close() + conn.commit() + conn.close() + except sqlite3.Error as e: + logging.error(str(e)) + conn.rollback() + return False + return True + + def grab_book_list(self, tag="小说", pageno=1) -> bool: + logging.info("start to grab tag {} page {}...".format(tag, pageno)) + self.save_current_progress(tag, pageno) + url = "https://book.douban.com/tag/{}?start={}&type=T".format(tag, pageno) + r = requests.get(url, headers=get_user_agent()) + r.encoding = "utf-8" + h: etree.Element = etree.HTML(r.text) + + li_list: [] = h.xpath( + '/html/body/div[@id="wrapper"]/div[@id="content"]' + '/div[@class="grid-16-8 clearfix"]' + '/div[@class="article"]/div[@id="subject_list"]' + '/ul/li/div[@class="info"]/h2/a/@href' + ) + next_page = h.xpath( + '/html/body/div[@id="wrapper"]/div[@id="content"]' + '/div[@class="grid-16-8 clearfix"]' + '/div[@class="article"]/div[@id="subject_list"]' + '/div[@class="paginator"]/span[@class="next"]/a[@href]' + ) + has_next = True + if len(next_page) == 0: + has_next = False + if len(li_list) == 0: + return False + + for li in li_list: + li.strip("") + book_id = li.strip("/").split("/")[-1] + try: + delay = float(random.randint(0, 200)) / 100.0 + time.sleep(delay) + self.crow_book_info(book_id) + except BaseException as e: + logging.error( + logging.error("error when scrape {}, {}".format(book_id, str(e))) + ) + return has_next + + def get_tag_list(self) -> [str]: + ret = [] + conn = sqlite3.connect(self.database) + results = conn.execute( + "SELECT tags.tag from tags join progress where tags.tag >= progress.tag" + ) + for row in results: + ret.append(row[0]) + return ret + + def crow_book_info(self, book_id) -> bool: + conn = sqlite3.connect(self.database) + for _ in conn.execute("SELECT id from book where id = ('{}')".format(book_id)): + return + + url = "https://book.douban.com/subject/{}/".format(book_id) + r = requests.get(url, headers=get_user_agent()) + r.encoding = "utf-8" + h: etree.Element = etree.HTML(r.text) + e_text = h.xpath('/html/body/div[@id="wrapper"]/h1/span/text()') + if len(e_text) == 0: + return False + + title = e_text[0] + + elements = h.xpath( + '/html/body/div[@id="wrapper"]' + '/div[@id="content"]/div[@class="grid-16-8 clearfix"]' + '/div[@class="article"]' + ) + if len(elements) == 0: + return False + + e_article = elements[0] + + book_intro = "" + author_intro = "" + content = "" + tags = "" + + e_book_intro = e_article.xpath( + 'div[@class="related_info"]' + '/div[@class="indent"][@id="link-report"]/*' + '/div[@class="intro"]/*/text()' + ) + for line in e_book_intro: + line = line.strip() + if line != "": + book_intro = book_intro + line + "\n" + + e_author_intro = e_article.xpath( + 'div[@class="related_info"]' + '/div[@class="indent "]/*' + '/div[@class="intro"]/*/text()' + ) + for line in e_author_intro: + line = line.strip() + if line != "": + author_intro = author_intro + line + "\n" + + e_content = e_article.xpath( + 'div[@class="related_info"]' + '/div[@class="indent"][@id="dir_' + book_id + '_full"]/text()' + ) + for line in e_content: + line = line.strip() + if line != "": + content = content + line + "\n" + + e_tags = e_article.xpath( + 'div[@class="related_info"]/' + 'div[@id="db-tags-section"]/' + 'div[@class="indent"]/span/a/text()' + ) + for line in e_tags: + line = line.strip() + if line != "": + tags = tags + line + "\n" + + e_subject = e_article.xpath( + 'div[@class="indent"]' + '/div[@class="subjectwrap clearfix"]' + '/div[@class="subject clearfix"]' + ) + pic_href = e_subject[0].xpath('div[@id="mainpic"]/a/@href') + picture = None + if len(pic_href) > 0: + res = requests.get(pic_href[0], headers=get_user_agent()) + picture = res.content + + info_children = e_subject[0].xpath('div[@id="info"]/child::node()') + + e_array = [] + e_dict = dict() + + for e in info_children: + if isinstance(e, etree._ElementUnicodeResult): + e_dict["text"] = e + elif isinstance(e, etree._Element): + if e.tag == "br": + e_array.append(e_dict) + e_dict = dict() + else: + e_dict[e.tag] = e + + book_info = dict() + for d in e_array: + label = "" + span = d.get("span") + a_label = span.xpath("span/text()") + if len(a_label) > 0 and label == "": + label = a_label[0].strip() + a_label = span.xpath("text()") + if len(a_label) > 0 and label == "": + label = a_label[0].strip() + label = label.strip(":") + text = d.get("text").strip() + e_a = d.get("a") + text.strip() + text.strip(":") + if label == "作者" or label == "译者": + a = span.xpath("a/text()") + if text == "" and len(a) == 1: + text = a[0].strip() + if text == "" and e_a is not None: + text_a = e_a.xpath("text()") + if len(text_a) > 0: + text = text_a[0].strip() + text = re.sub(r"\s+", " ", text) + if text != "": + book_info[label] = text + + sql = ( + "INSERT INTO book(" + "id, title, author, " + "publisher, original_title, translator, " + "pub_year, pages, price, " + "currency_unit, binding, isbn, " + "author_intro, book_intro, content, " + "tags, picture)" + "VALUES(" + "?, ?, ?, " + "?, ?, ?, " + "?, ?, ?, " + "?, ?, ?, " + "?, ?, ?, " + "?, ?)" + ) + + unit = None + price = None + pages = None + conn = sqlite3.connect(self.database) + try: + s_price = book_info.get("定价") + if s_price is None: + # price cannot be NULL + logging.error( + "error when scrape book_id {}, cannot retrieve price...", book_id + ) + return None + else: + e = re.findall(self.pattern_number, s_price) + if len(e) != 0: + number = e[0] + unit = s_price.replace(number, "").strip() + price = int(float(number) * 100) + + s_pages = book_info.get("页数") + if s_pages is not None: + # pages can be NULL + e = re.findall(self.pattern_number, s_pages) + if len(e) != 0: + pages = int(e[0]) + + conn.execute( + sql, + ( + book_id, + title, + book_info.get("作者"), + book_info.get("出版社"), + book_info.get("原作名"), + book_info.get("译者"), + book_info.get("出版年"), + pages, + price, + unit, + book_info.get("装帧"), + book_info.get("ISBN"), + author_intro, + book_intro, + content, + tags, + picture, + ), + ) + conn.commit() + except sqlite3.Error as e: + logging(str(e)) + conn.rollback() + except TypeError as e: + logging.error("error when scrape {}, {}".format(book_id, str(e))) + conn.rollback() + return False + conn.close() + return True + + +if __name__ == "__main__": + scraper = Scraper() + scraper.start_grab() diff --git a/fe/test/gen_book_data.py b/fe/test/gen_book_data.py new file mode 100644 index 0000000..4bda418 --- /dev/null +++ b/fe/test/gen_book_data.py @@ -0,0 +1,57 @@ +import random +from fe.access import book +from fe.access.new_seller import register_new_seller + + +class GenBook: + def __init__(self, user_id, store_id): + self.user_id = user_id + self.store_id = store_id + self.password = self.user_id + self.seller = register_new_seller(self.user_id, self.password) + code = self.seller.create_store(store_id) + assert code == 200 + self.__init_book_list__() + + def __init_book_list__(self): + self.buy_book_info_list = [] + self.buy_book_id_list = [] + + def gen(self, non_exist_book_id: bool, low_stock_level, max_book_count: int = 100) -> (bool, []): + self.__init_book_list__() + ok = True + book_db = book.BookDB() + rows = book_db.get_book_count() + start = 0 + if rows > max_book_count: + start = random.randint(0, rows - max_book_count) + size = random.randint(1, max_book_count) + books = book_db.get_book_info(start, size) + book_id_exist = [] + book_id_stock_level = {} + for bk in books: + if low_stock_level: + stock_level = random.randint(0, 100) + else: + stock_level = random.randint(2, 100) + code = self.seller.add_book(self.store_id, stock_level, bk) + assert code == 200 + book_id_stock_level[bk.id] = stock_level + book_id_exist.append(bk) + + for bk in book_id_exist: + stock_level = book_id_stock_level[bk.id] + if stock_level > 1: + buy_num = random.randint(1, stock_level) + else: + buy_num = 0 + # add a new pair + if non_exist_book_id: + bk.id = bk.id + "_x" + if low_stock_level: + buy_num = stock_level + 1 + self.buy_book_info_list.append((bk, buy_num)) + + for item in self.buy_book_info_list: + self.buy_book_id_list.append((item[0].id, item[1])) + return ok, self.buy_book_id_list diff --git a/fe/test/test.md b/fe/test/test.md new file mode 100644 index 0000000..c7fe16e --- /dev/null +++ b/fe/test/test.md @@ -0,0 +1 @@ +add functionality test here diff --git a/fe/test/test_add_book.py b/fe/test/test_add_book.py new file mode 100644 index 0000000..8491c4d --- /dev/null +++ b/fe/test/test_add_book.py @@ -0,0 +1,51 @@ +import pytest + +from fe.access.new_seller import register_new_seller +from fe.access import book +import uuid + + +class TestAddBook: + @pytest.fixture(autouse=True) + def pre_run_initialization(self): + # do before test + self.seller_id = "test_add_books_seller_id_{}".format(str(uuid.uuid1())) + self.store_id = "test_add_books_store_id_{}".format(str(uuid.uuid1())) + self.password = self.seller_id + self.seller = register_new_seller(self.seller_id, self.password) + + code = self.seller.create_store(self.store_id) + assert code == 200 + book_db = book.BookDB() + self.books = book_db.get_book_info(0, 2) + + yield + # do after test + + def test_ok(self): + for b in self.books: + code = self.seller.add_book(self.store_id, 0, b) + assert code == 200 + + def test_error_non_exist_store_id(self): + for b in self.books: + # non exist store id + code = self.seller.add_book(self.store_id + "x", 0, b) + assert code != 200 + + def test_error_exist_book_id(self): + for b in self.books: + code = self.seller.add_book(self.store_id, 0, b) + assert code == 200 + for b in self.books: + # exist book id + code = self.seller.add_book(self.store_id, 0, b) + assert code != 200 + + def test_error_non_exist_user_id(self): + for b in self.books: + # non exist user id + self.seller.seller_id = self.seller.seller_id + "_x" + code = self.seller.add_book(self.store_id, 0, b) + assert code != 200 + diff --git a/fe/test/test_add_funds.py b/fe/test/test_add_funds.py new file mode 100644 index 0000000..de5b525 --- /dev/null +++ b/fe/test/test_add_funds.py @@ -0,0 +1,29 @@ +import pytest +import uuid +from fe.access.new_buyer import register_new_buyer + + +class TestAddFunds: + @pytest.fixture(autouse=True) + def pre_run_initialization(self): + self.user_id = "test_add_funds_{}".format(str(uuid.uuid1())) + self.password = self.user_id + self.buyer = register_new_buyer(self.user_id, self.password) + yield + + def test_ok(self): + code = self.buyer.add_funds(1000) + assert code == 200 + + code = self.buyer.add_funds(-1000) + assert code == 200 + + def test_error_user_id(self): + self.buyer.user_id = self.buyer.user_id + "_x" + code = self.buyer.add_funds(10) + assert code != 200 + + def test_error_password(self): + self.buyer.password = self.buyer.password + "_x" + code = self.buyer.add_funds(10) + assert code != 200 diff --git a/fe/test/test_add_stock_level.py b/fe/test/test_add_stock_level.py new file mode 100644 index 0000000..ea61f06 --- /dev/null +++ b/fe/test/test_add_stock_level.py @@ -0,0 +1,46 @@ +import pytest +from fe.access.new_seller import register_new_seller +from fe.access import book +import uuid + + +class TestAddStockLevel: + @pytest.fixture(autouse=True) + def pre_run_initialization(self): + self.user_id = "test_add_book_stock_level1_user_{}".format(str(uuid.uuid1())) + self.store_id = "test_add_book_stock_level1_store_{}".format(str(uuid.uuid1())) + self.password = self.user_id + self.seller = register_new_seller(self.user_id, self.password) + + code = self.seller.create_store(self.store_id) + assert code == 200 + book_db = book.BookDB() + self.books = book_db.get_book_info(0, 5) + for bk in self.books: + code = self.seller.add_book(self.store_id, 0, bk) + assert code == 200 + yield + + def test_error_user_id(self): + for b in self.books: + book_id = b.id + code = self.seller.add_stock_level(self.user_id + "_x", self.store_id, book_id, 10) + assert code != 200 + + def test_error_store_id(self): + for b in self.books: + book_id = b.id + code = self.seller.add_stock_level(self.user_id, self.store_id + "_x", book_id, 10) + assert code != 200 + + def test_error_book_id(self): + for b in self.books: + book_id = b.id + code = self.seller.add_stock_level(self.user_id, self.store_id, book_id + "_x", 10) + assert code != 200 + + def test_ok(self): + for b in self.books: + book_id = b.id + code = self.seller.add_stock_level(self.user_id, self.store_id, book_id, 10) + assert code == 200 diff --git a/fe/test/test_bench.py b/fe/test/test_bench.py new file mode 100644 index 0000000..969c18b --- /dev/null +++ b/fe/test/test_bench.py @@ -0,0 +1,8 @@ +from fe.bench.run import run_bench + + +def test_bench(): + try: + run_bench() + except Exception as e: + assert 200==100,"test_bench过程出现异常" \ No newline at end of file diff --git a/fe/test/test_create_store.py b/fe/test/test_create_store.py new file mode 100644 index 0000000..fc48eb0 --- /dev/null +++ b/fe/test/test_create_store.py @@ -0,0 +1,26 @@ +import pytest + +from fe.access.new_seller import register_new_seller +import uuid + + +class TestCreateStore: + @pytest.fixture(autouse=True) + def pre_run_initialization(self): + self.user_id = "test_create_store_user_{}".format(str(uuid.uuid1())) + self.store_id = "test_create_store_store_{}".format(str(uuid.uuid1())) + self.password = self.user_id + yield + + def test_ok(self): + self.seller = register_new_seller(self.user_id, self.password) + code = self.seller.create_store(self.store_id) + assert code == 200 + + def test_error_exist_store_id(self): + self.seller = register_new_seller(self.user_id, self.password) + code = self.seller.create_store(self.store_id) + assert code == 200 + + code = self.seller.create_store(self.store_id) + assert code != 200 diff --git a/fe/test/test_login.py b/fe/test/test_login.py new file mode 100644 index 0000000..ee5a118 --- /dev/null +++ b/fe/test/test_login.py @@ -0,0 +1,39 @@ +import time + +import pytest + +from fe.access import auth +from fe import conf + + +class TestLogin: + @pytest.fixture(autouse=True) + def pre_run_initialization(self): + self.auth = auth.Auth(conf.URL) + # register a user + self.user_id = "test_login_{}".format(time.time()) + self.password = "password_" + self.user_id + self.terminal = "terminal_" + self.user_id + assert self.auth.register(self.user_id, self.password) == 200 + yield + + def test_ok(self): + code, token = self.auth.login(self.user_id, self.password, self.terminal) + assert code == 200 + + code = self.auth.logout(self.user_id + "_x", token) + assert code == 401 + + code = self.auth.logout(self.user_id, token + "_x") + assert code == 401 + + code = self.auth.logout(self.user_id, token) + assert code == 200 + + def test_error_user_id(self): + code, token = self.auth.login(self.user_id + "_x", self.password, self.terminal) + assert code == 401 + + def test_error_password(self): + code, token = self.auth.login(self.user_id, self.password + "_x", self.terminal) + assert code == 401 diff --git a/fe/test/test_new_order.py b/fe/test/test_new_order.py new file mode 100644 index 0000000..288bc57 --- /dev/null +++ b/fe/test/test_new_order.py @@ -0,0 +1,48 @@ +import pytest + +from fe.test.gen_book_data import GenBook +from fe.access.new_buyer import register_new_buyer +import uuid + + +class TestNewOrder: + @pytest.fixture(autouse=True) + def pre_run_initialization(self): + self.seller_id = "test_new_order_seller_id_{}".format(str(uuid.uuid1())) + self.store_id = "test_new_order_store_id_{}".format(str(uuid.uuid1())) + self.buyer_id = "test_new_order_buyer_id_{}".format(str(uuid.uuid1())) + self.password = self.seller_id + self.buyer = register_new_buyer(self.buyer_id, self.password) + self.gen_book = GenBook(self.seller_id, self.store_id) + yield + + def test_non_exist_book_id(self): + ok, buy_book_id_list = self.gen_book.gen(non_exist_book_id=True, low_stock_level=False) + assert ok + code, _ = self.buyer.new_order(self.store_id, buy_book_id_list) + assert code != 200 + + def test_low_stock_level(self): + ok, buy_book_id_list = self.gen_book.gen(non_exist_book_id=False, low_stock_level=True) + assert ok + code, _ = self.buyer.new_order(self.store_id, buy_book_id_list) + assert code != 200 + + def test_ok(self): + ok, buy_book_id_list = self.gen_book.gen(non_exist_book_id=False, low_stock_level=False) + assert ok + code, _ = self.buyer.new_order(self.store_id, buy_book_id_list) + assert code == 200 + + def test_non_exist_user_id(self): + ok, buy_book_id_list = self.gen_book.gen(non_exist_book_id=False, low_stock_level=False) + assert ok + self.buyer.user_id = self.buyer.user_id + "_x" + code, _ = self.buyer.new_order(self.store_id, buy_book_id_list) + assert code != 200 + + def test_non_exist_store_id(self): + ok, buy_book_id_list = self.gen_book.gen(non_exist_book_id=False, low_stock_level=False) + assert ok + code, _ = self.buyer.new_order(self.store_id + "_x", buy_book_id_list) + assert code != 200 diff --git a/fe/test/test_password.py b/fe/test/test_password.py new file mode 100644 index 0000000..8ef999d --- /dev/null +++ b/fe/test/test_password.py @@ -0,0 +1,47 @@ +import uuid + +import pytest + +from fe.access import auth +from fe import conf + + +class TestPassword: + @pytest.fixture(autouse=True) + def pre_run_initialization(self): + self.auth = auth.Auth(conf.URL) + # register a user + self.user_id = "test_password_{}".format(str(uuid.uuid1())) + self.old_password = "old_password_" + self.user_id + self.new_password = "new_password_" + self.user_id + self.terminal = "terminal_" + self.user_id + + assert self.auth.register(self.user_id, self.old_password) == 200 + yield + + def test_ok(self): + code = self.auth.password(self.user_id, self.old_password, self.new_password) + assert code == 200 + + code, new_token = self.auth.login(self.user_id, self.old_password, self.terminal) + assert code != 200 + + code, new_token = self.auth.login(self.user_id, self.new_password, self.terminal) + assert code == 200 + + code = self.auth.logout(self.user_id, new_token) + assert code == 200 + + def test_error_password(self): + code = self.auth.password(self.user_id, self.old_password + "_x", self.new_password) + assert code != 200 + + code, new_token = self.auth.login(self.user_id, self.new_password, self.terminal) + assert code != 200 + + def test_error_user_id(self): + code = self.auth.password(self.user_id + "_x", self.old_password, self.new_password) + assert code != 200 + + code, new_token = self.auth.login(self.user_id, self.new_password, self.terminal) + assert code != 200 diff --git a/fe/test/test_payment.py b/fe/test/test_payment.py new file mode 100644 index 0000000..e7f54b6 --- /dev/null +++ b/fe/test/test_payment.py @@ -0,0 +1,70 @@ +import pytest + +from fe.access.buyer import Buyer +from fe.test.gen_book_data import GenBook +from fe.access.new_buyer import register_new_buyer +from fe.access.book import Book +import uuid + + +class TestPayment: + seller_id: str + store_id: str + buyer_id: str + password:str + buy_book_info_list: [Book] + total_price: int + order_id: str + buyer: Buyer + + @pytest.fixture(autouse=True) + def pre_run_initialization(self): + self.seller_id = "test_payment_seller_id_{}".format(str(uuid.uuid1())) + self.store_id = "test_payment_store_id_{}".format(str(uuid.uuid1())) + self.buyer_id = "test_payment_buyer_id_{}".format(str(uuid.uuid1())) + self.password = self.seller_id + gen_book = GenBook(self.seller_id, self.store_id) + ok, buy_book_id_list = gen_book.gen(non_exist_book_id=False, low_stock_level=False, max_book_count=5) + self.buy_book_info_list = gen_book.buy_book_info_list + assert ok + b = register_new_buyer(self.buyer_id, self.password) + self.buyer = b + code, self.order_id = b.new_order(self.store_id, buy_book_id_list) + assert code == 200 + self.total_price = 0 + for item in self.buy_book_info_list: + book: Book = item[0] + num = item[1] + if book.price is None: + continue + else: + self.total_price = self.total_price + book.price * num + yield + + def test_ok(self): + code = self.buyer.add_funds(self.total_price) + assert code == 200 + code = self.buyer.payment(self.order_id) + assert code == 200 + + def test_authorization_error(self): + code = self.buyer.add_funds(self.total_price) + assert code == 200 + self.buyer.password = self.buyer.password + "_x" + code = self.buyer.payment(self.order_id) + assert code != 200 + + def test_not_suff_funds(self): + code = self.buyer.add_funds(self.total_price - 1) + assert code == 200 + code = self.buyer.payment(self.order_id) + assert code != 200 + + def test_repeat_pay(self): + code = self.buyer.add_funds(self.total_price) + assert code == 200 + code = self.buyer.payment(self.order_id) + assert code == 200 + + code = self.buyer.payment(self.order_id) + assert code != 200 diff --git a/fe/test/test_register.py b/fe/test/test_register.py new file mode 100644 index 0000000..42acc5e --- /dev/null +++ b/fe/test/test_register.py @@ -0,0 +1,43 @@ +import time + +import pytest + +from fe.access import auth +from fe import conf + + +class TestRegister: + @pytest.fixture(autouse=True) + def pre_run_initialization(self): + self.user_id = "test_register_user_{}".format(time.time()) + self.password = "test_register_password_{}".format(time.time()) + self.auth = auth.Auth(conf.URL) + yield + + def test_register_ok(self): + code = self.auth.register(self.user_id, self.password) + assert code == 200 + + def test_unregister_ok(self): + code = self.auth.register(self.user_id, self.password) + assert code == 200 + + code = self.auth.unregister(self.user_id, self.password) + assert code == 200 + + def test_unregister_error_authorization(self): + code = self.auth.register(self.user_id, self.password) + assert code == 200 + + code = self.auth.unregister(self.user_id + "_x", self.password) + assert code != 200 + + code = self.auth.unregister(self.user_id, self.password + "_x") + assert code != 200 + + def test_register_error_exist_user_id(self): + code = self.auth.register(self.user_id, self.password) + assert code == 200 + + code = self.auth.register(self.user_id, self.password) + assert code != 200 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..f25106e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +simplejson +lxml +codecov +coverage +flask +pre-commit +pytest +PyJWT +requests diff --git a/script/test.sh b/script/test.sh new file mode 100644 index 0000000..ae7bfb0 --- /dev/null +++ b/script/test.sh @@ -0,0 +1,6 @@ +#!/bin/sh +export PATHONPATH=`pwd` +coverage run --timid --branch --source fe,be --concurrency=thread -m pytest -v --ignore=fe/data +coverage combine +coverage report +coverage html diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..3910635 --- /dev/null +++ b/setup.py @@ -0,0 +1,22 @@ +import setuptools + +with open("README.md", "r") as fh: + long_description = fh.read() + +setuptools.setup( + name="bookstore", + version="0.0.1", + author="DaSE-DBMS", + author_email="DaSE-DBMS@DaSE-DBMS.com", + description="Buy Books Online", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/DaSE-DBMS/bookstore.git", + packages=setuptools.find_packages(), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ], + python_requires=">=3.6", +)