import pandas as pd
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split

data = pd.read_csv('./data/heart.csv')
# the csv url: https://github.com/hivandu/colab/blob/master/AI_Data/data/heart.csv

# Print a brief summary of the data set

data.info()
data.shape

data.target.value_counts()

# Change the "sex" column into two columns "sex_0" and "sex_1"
sex = pd.get_dummies(data['sex'], prefix = 'sex')        

# Add "sex_0" and "sex_1" to the data set. 
data = pd.concat([data, sex], axis = 1)


# And delete the sex column.
data = data.drop(columns = ['sex'])


# Print out the first five lines. Check whether sex_0, sex_1 are added successfully, and whether sex is deleted successfully.
data.head()

# Get sample label
data_y = data.target.values
data_y.shape

# Get sample feature set
data_x = data.drop(['target'], axis = 1)
data_x.shape

# Divide the data set
train_x, test_x, train_y, test_y = train_test_split(data_x, data_y, test_size = 0.3, random_state=33)

# initialize
ss = StandardScaler()

# The fit function/module is used to train model parameters
ss.fit(train_x)

# Standardize the training set and test set
train_x = ss.transform(train_x)
test_x = ss.transform(test_x)

# Define a logistic regression model
lr = LogisticRegression()
lr.fit(train_x, train_y)

# Calculate the training set score
lr.score(train_x, train_y)

# Calculate test set score
lr.score(test_x, test_y)

# Use the classification_report function to display a text report of the main classification indicators
predict = lr.predict(test_x)
print(classification_report(test_y, predict))

def fac(n): # return n!
    if n == 1: 
        return 1 # 返回操作
    else:
        return n * fac(n-1) # 乘法操作 + 返回操作 + 函数调用

fac(1)
> 1

fac(100)
> 93326215443944152681699238856266700490715968264381621468592963895217599993229915608941463976156518286253697920827223758251185210916864000000000000000000000000

fac_100 = """93326215443944152681699238856266700490715968264381621468592963895217599993229915608941463976156518286253697920827223758251185210916864000000000000000000000000"""

len(fac_100)
> 158

?? N --> fac(n)
# 乘法操作 + 返回操作 + 函数调用
?? (N - 1)--> fac(n-1)
?? N == 100  fac(N) 
??? 99

# import
import codecs
import os
import jieba

data_train = './data/cnews.train.txt' # training data file name  
data_test = './data/cnews.test.txt'  # test data file name
vocab = './data/cnews.vocab.txt' # dictionary

with codecs.open(data_train, 'r', 'utf-8') as f:
    lines = f.readlines()

# print sample content
label, content = lines[0].strip('\r\n').split('\t')
content

# print word segment results
segment = jieba.cut(content)
print('/'.join(segment))

# cut data
def process_line(idx, line):
    data = tuple(line.strip('\r\n').split('\t'))
    if not len(data)==2:
        return None
    content_segged = list(jieba.cut(data[1]))
    if idx % 1000 == 0:
        print('line number: {}'.format(idx))
    return (data[0], content_segged)
    
# data loading method
def load_data(file):
    with codecs.open(file, 'r', 'utf-8') as f:
        lines = f.readlines()
    data_records = [process_line(idx, line) for idx, line in enumerate(lines)]
    data_records = [data for data in data_records if data is not None]
    return data_records

# load and process training data
train_data = load_data(data_train)
print('first training data: label {} segment {}'.format(train_data[0][0], '/'.join(train_data[0][1])))
# load and process testing data
test_data = load_data(data_test)
print('first testing data: label {} segment {}'.format(test_data[0][0], '/'.join(test_data[0][1])))

def build_vocab(train_data, thresh):
    vocab = {'<UNK>': 0}
    word_count = {} # word frequency
    for idx, data in enumerate(train_data):
        content = data[1]
        for word in content:
            if word in word_count:
                word_count[word] += 1
            else:
                word_count[word] = 1
    word_list = [(k, v) for k, v in word_count.items()]
    print('word list length: {}'.format(len(word_list)))
    word_list.sort(key = lambda x : x[1], reverse = True) # sorted by word frequency
    word_list_filtered = [word for word in word_list if word[1] > thresh]
    print('word list length after filtering: {}'.format(len(word_list_filtered)))
    # construct vocab
    for word in word_list_filtered:
        vocab[word[0]] = len(vocab)
    print('vocab size: {}'.format(len(vocab))) # vocab size is word list size +1 due to unk token
    return vocab

vocab = build_vocab(train_data, 1)

def build_label_vocab(cate_file):
    label_vocab = {}
    with codecs.open(cate_file, 'r', 'utf-8') as f:
        for lines in f:
            line = lines.strip().split('\t')
            label_vocab[line[0]]  = int(line[1])
    return label_vocab

label_vocab = build_label_vocab('./data/cnews.category.txt')
print(f'label vocab: {label_vocab}')

def construct_trainable_matrix(corpus, vocab, label_vocab, out_file):
    records = []
    for idx, data in enumerate(corpus):
        if idx % 1000 == 0:
            print('process {} data'.format(idx))
        label = str(label_vocab[data[0]]) # label id
        token_dict = {}
        for token in data[1]:
            token_id = vocab.get(token, 0)
            if token_id in token_dict:
                token_dict[token_id] += 1
            else:
                token_dict[token_id] = 1
        feature = [str(int(k) + 1) + ':' + str(v) for k,v in token_dict.items()]
        feature_text = ' '.join(feature)
        records.append(label + ' ' + feature_text)
    
    with open(out_file, 'w') as f:
        f.write('\n'.join(records))

construct_trainable_matrix(train_data, vocab, label_vocab, './data/train.svm.txt')
construct_trainable_matrix(test_data, vocab, label_vocab, './data/test.svm.txt')

from libsvm import svm
from libsvm.svmutil import svm_read_problem,svm_train,svm_predict,svm_save_model,svm_load_model

# train svm
train_label, train_feature = svm_read_problem('./data/train.svm.txt')
print(train_label[0], train_feature[0])
model=svm_train(train_label,train_feature,'-s 0 -c 5 -t 0 -g 0.5 -e 0.1')

# predict
test_label, test_feature = svm_read_problem('./data/test.svm.txt')
print(test_label[0], test_feature[0])
p_labs, p_acc, p_vals = svm_predict(test_label, test_feature, model)

print('accuracy: {}'.format(p_acc))

from selenium import webdriver
import time
driver = webdriver.Chrome('/Applications/chromedriver')

# login weibo
def weibo_login(username, password):

    # open weibo index
    driver.get('https://passport.weibo.cn/signin/login')
    driver.implicitly_wait(5)
    time.sleep(1)

    # fill the info: username, password
    driver.find_element_by_id('loginName').send_keys(username)
    driver.find_element_by_id('loginPassword').send_keys(password)
    time.sleep(1)

    # click login
    driver.find_element_by_id('loginAction').click()
    time.sleep(1)

# set username, password
username = 'ivandoo75@gmail.com'
password = 'ooxx'

# Mobile phone verification is required here, but still can’t log in fully automatically
weibo_login(username, password)

def add_follow(uid):
    driver.get('https://m.weibo.com/u/' + str(uid))
    time.sleep(1)

    # driver.find_element_by_id('follow').click()
    follow_button = driver.find_element_by_xpath('//div[@class="btn_bed W_fl"]')
    follow_button.click()
    time.sleep(1)

    # select group
    group_button = driver.find_element_by_xpath('//div[@class="list_content W_f14"]/ul[@class="list_ul"]/li[@class="item"][2]')
    group_button.click()
    time.sleep(1)

    # cancel the select
    cancel_button = driver.find_element_by_xpath('//div[@class="W_layer_btn S_bg1"]/a[@class="W_btn_b btn_34px"]')
    cancel_button.click()
    time.sleep(1)

# 每天学点心理学 UID
uid = '1890826225'
add_follow(uid)

def add_comment(weibo_url, content):
    driver.get(weibo_url)
    driver.implicitly_wait(5)

    content_textarea = driver.find_element_by_css_selector('textarea.W.input').clear()
    content_textarea = driver.find_element_by_css_selector('textarea.W.input').send_keys(content)

    time.sleep(2)

    comment_button = driver.find_element_by_css_selector('.W_btn_a').click()

# post the text
def post_weibo(content):
    # go to the user index
    driver.get('https://weibo.com')
    driver.implicitly_wait(5)

    # click publish button
    # post_button = driver.find_element_by_css_selector('[node-type="publish"]').click()

    # input content word to textarea
    content_textarea = driver.find_element_by_css_selector('textarea.W_input[node-type="textEl"]').send_keys(content)
    time.sleep(2)

    # click publish button
    post_button = driver.find_element_by_css_selector("[node-type='submit']").click()
    time.sleep(1)

# comment the weibo
weibo_url = 'https://weibo.com/1890826225/HjjqSahwl'
content= 'here is Hivan du, Best wish to u.'

# auto send weibo
content = 'Learning is a belief!'
post_weibo(content)

# Import package
# Used to load the Boston housing price data set
from sklearn.datasets import load_boston
# pandas toolkit If you are unfamiliar with pandas, you can refer to the official 10-minute tutorial: https://pandas.pydata.org/pandas-docs/stable/10min.html
import pandas as pd
import numpy as np
# seaborn for drawing
import seaborn as sns
import matplotlib.pyplot as plt
# Show drawing
%matplotlib inline


data = load_boston() # load datase

data.keys() # Fields inside data

df = pd.DataFrame(data['data'])

# Looking at the first 5 rows of the dataframe, we can see that the column names are numbers
df.head(5)

data['feature_names'] # Feature name

# Replace numeric column names with feature names
df.columns = data['feature_names']
df.head(5)

# The target is the house price, which is also our target value. We assign the target value to the dataframe
df['price'] = data['target']
df.head(5)

# View the correlation coefficient between the feature and price, positive correlation and negative correlation
sns.heatmap(df.corr(), annot=True, fmt='.1f')

plt.scatter(df['RM'], df['price'])


plt.figure(figsize=(20, 5))

# View the data distribution display of some features and price
features = ['LSTAT', 'RM']
target = df['price']

for i, col in enumerate(features):
    plt.subplot(1, len(features), i+1)
    x = df[col]
    y = target
    plt.scatter(x, y, marker = 'o')
    plt.title('{} price'.format(col))
    plt.xlabel(col)
    plt.ylabel('price')


# Simple example: univariate forecast price
x = df['RM']
y = df['price']

history_notes = {_x: _y for _x, _y in zip(x,y)}

history_notes[6.575]


# Find the top three prices that are closest to RM:6.57,
similary_ys = [y for _, y in sorted(history_notes.items(), key=lambda x_y: (x_y[0] - 6.57) ** 2)[:3]]
similary_ys


# Calculate the average of three
np.mean(similary_ys)

def knn(query_x, history, top_n = 3):
    sorted_notes = sorted(history.items(), key = lambda x_y: (x_y[0] - query_x)**2) 
    similar_notes = sorted_notes[:top_n]
    similar_ys = [y for _, y in similar_notes]

    return np.mean(similar_ys)

knn(5.4, history_notes)

def loss(y_hat, y):
    return np.mean((y_hat - y)**2)

import random

min_loss = float('inf')

best_k, best_b = None, None


for step in range(1000):
    min_v, max_v = -100, 100
    k, b = random.randrange(min_v, max_v), random.randrange(min_v, max_v)
    y_hats = [k * rm_i + b for rm_i in x]
    current_loss = loss(y_hats, y)

    if current_loss < min_loss:
        min_loss = current_loss
        best_k, best_b = k, b
        print(f'{step}, we have func f(rm) = {k} * rm + {b},  lss is :{current_loss}')

plt.scatter(x, y)
plt.scatter(x, [best_k * rm + best_b for rm in x])

def partial_k(k, b, x, y):
    return 2  * np.mean((k*x+b-y) * x)

def partial_b(k, b, x, y):
    return 2 * np.mean(k*x+b-y)

k, b = random.random(), random.random()
min_loss = float('inf')

best_k, best_b = None, None
learning_rate = 1e-2

for step in range(2000):
    k, b = k + (-1 * partial_k(k, b, x, y) * learning_rate), b + (-1 * partial_b(k, b, x, y) * learning_rate)
    y_hats = k * x + b
    current_loss = loss(y_hats, y)

    if current_loss < min_loss:
        min_loss = current_loss
        best_k, best_b = k, b
        print(f'setp {step}, we have func f(rm) = {k} * rm + {b},  lss is :{current_loss}')

best_k, best_b


plt.scatter(x, y)
plt.scatter(x, [best_k * rm + best_b for rm in x])

def sigmoid(x):
    return 1 / (1+np.exp(-x))

sub_x = np.linspace(-10, 10)
plt.plot(sub_x, sigmoid(sub_x))


def random_linear(x):
    k, b = random.random(), random.random()
    return k * x + b

def complex_function(x):
    return (random_linear(x))

for _ in range(10):
    index = random.randrange(0, len(sub_x))
    sub_x_1, sub_x_2 = sub_x[:index], sub_x[index:]
    new_y = np.concatenate((complex_function(sub_x_1), complex_function(sub_x_2)))
    plt.plot(sub_x, new_y)

import numpy as np
import pandas as pd

from sklearn.datasets import load_boston
data = load_boston()
data.keys()

data_train = data.data
data_traget = data.target

df = pd.DataFrame(data_train, columns = data.feature_names)
df.head()

df.describe() # Data description, you can view the statistics of each variable

from sklearn.preprocessing import StandardScaler
# z = (x-u) / s u is the mean, s is the standard deviation
ss = StandardScaler() 
data_train = ss.fit_transform(data_train)
# For linear models, normalization or standardization is generally required, otherwise gradient explosion will occur, and tree models are generally not required
data_train = pd.DataFrame(data_train, columns = ['CRIM','ZN','INDUS','CHAS','NOX','RM','AGE','DIS','RAD','TAX','PTRATIO','B','LSTAT'])
data_train.describe() 

# y=Σwixi+
# Because the derivation of b is all 1, add a bias b to the data and set it to 1, as a feature of the data and update the gradient wi*b=wi
data_train['bias']  = 1
data_train

from sklearn.model_selection import train_test_split
train_x, test_x, train_y, test_y = train_test_split(data_train, data_traget, test_size = 0.2, random_state=42)

print('train_x.shape, train_y.shape', train_x.shape, train_y.shape)
print('test_x.shape, test_y.shape', test_x.shape, test_y.shape)

train_x = np.array(train_x)

def l1_cost(x, y, theta):
    """
    x: 特征
    y: 目标值
    thta: 模型参数
    """
    k = x.shape[0]
    total_cost = 0
    for i in range(k):
        total_cost += 1/k * np.abs(y[i] -theta.dot(x[i, :]))
    return total_cost

def l2_cost(x, y, theta):
    k = x.shape[0]
    total_cost = 0
    for i in range(k):
        total_cost += 1/k * (y[i] -theta.dot(x[i,:])) ** 2
    return total_cost

np.zeros(10).shape

def step_l1_gradient(x, y, learning_rate, theta):
    """
    Function to calculate the gradient of the MAE loss function
    Return the gradient value 0 for the non-differentiable point at 0
    X:特征向量
    y：目标值
    learing_rate:学习率
    theta:参数
    """
    n = x.shape[0]
        # print(n)
    e = y - x @ theta
    gradients = - (x.T @ np.sign(e)) / n # sign is a sign function
    thata = theta - learning_rate * gradients
    return theta

def step_l2_gradient(x, y, learning_rate, theta):
    k = x.shape[0]
    n = x.shape[1]
    gradients = np.zeros(n)
    for i in range(k):
        for j in range(n):
            gradients[j] += (-2/k) * (y[i] - (theta.dot(x[i, :]))) * x[i, j]
    theta = theta - learning_rate * gradient
    return theta

# def step_gradient(X, y, learning_rate, theta):
#     """
#     X:特征向量
#     y：目标值
#     learing_rate:学习率
#     theta:参数
#     """
#     m_deriv = 0
#     N = len(X)
#     for i in range(N):
#         # 计算偏导
#         # -x(y - (mx + b)) / |mx + b|
#         m_deriv += - X[i] * (y[i] - (theta*X[i] + b)) / abs(y[i] - (theta*X[i] + b))
#     # We subtract because the derivatives point in direction of steepest ascent
#     theta -= (m_deriv / float(N)) * learning_rate
# #     theta = theta - learning_rate * gradients
#     return theta

def gradient_descent(train_x, train_y, learning_rate, iterations):
    k = train_x.shape[0]
    n = train_x.shape[1]
    theta = np.zeros(n) # Initialization parameters

    loss_values = []
    # print(theta.shape)

    for i in range(iterations):
        theta = step_l1_gradient(train_x, train_y, learning_rate, theta)
        loss = l1_cost(train_x, train_y, theta)
        loss_values.append(loss)
        print(i, 'cost:', loss)
    return theta, loss_values

#  Training parameters
learning_rate = 0.04 # Learning rate
iterations = 300 # Number of iterations
theta, loss_values = gradient_descent(train_x, train_y, learning_rate, iterations)

# CART regression tree prediction
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split
from sklearn.datasets import load_boston
from sklearn.metrics import r2_score,mean_absolute_error, mean_squared_error
from sklearn.tree import DecisionTreeRegressor,export_graphviz
import graphviz

# Prepare data set
boston = load_boston()

# Explore data
print(boston.feature_names)

# Get feature set and price
features = boston.data
prices = boston.target


# Randomly extract 33% of the data as the test set, and the rest as the training set
train_features, test_features, train_price, test_price = train_test_split(features,prices,test_size=0.33)

# Create CART regression tree
dtr = DecisionTreeRegressor()

# Fitting and constructing CART regression tree
dtr.fit(train_features, train_price)

# Predict housing prices in the test set
predict_price = dtr.predict(test_features)

grap_data = export_graphviz(dtr, out_file=None)
graph = graphviz.Source(grap_data)

# Result evaluation of test set
print(f'Regression tree mean squared deviation:',mean_squared_error(test_price, predict_price))
print(f'Regression tree absolute value deviation mean:',mean_absolute_error(test_price, predict_price))

# Generate regression tree visualization
graph.render('Boston')

import pylab
import numpy as np
from scipy import signal

# set img
img = np.array([[10, 10, 10, 10, 10],[10, 5, 5, 5, 10], [10, 5, 5, 5, 10], [10, 5, 5, 5, 10], [10, 10, 10, 10, 10]])

# set convolution
fil = np.array([[-1, -1, 0], [-1, 0, 1], [0, 1, 1]])

# convolution the img
res = signal.convolve2d(img, fil, mode='valid')

# output the result
print(res)

import matplotlib.pyplot as plt
import pylab
import cv2
import numpy as np
from scipy import signal

# read the image
img = cv2.imread('./data/weixin.jpg', 0) # Any picture

# show the image
plt.imshow(img, cmap='gray')
pylab.show()

# set the convolution
fil = np.array([[-1,-1,0], [-1, 0, 1], [0, 1, 1]])

# convolution operation
res = signal.convolve2d(img, fil, mode='valid')
print(res)

# show convolution image
plt.imshow(res, cmap = 'gray')
pylab.show()

import keras
from keras.datasets import mnist
from keras.layers import Conv2D, MaxPooling2D
from keras.layers import Dense, Flatten
from keras.models import Sequential
import warnings
warnings.filterwarnings('ignore')

# load data
(train_x, train_y), (test_x, test_y) = mnist.load_data()

train_x = train_x.reshape(train_x.shape[0], 28, 28, 1)
test_x = test_x.reshape(test_x.shape[0], 28, 28, 1)
train_x = train_x / 255
test_x = test_x / 255

train_y = keras.utils.to_categorical(train_y, 10)
test_y = keras.utils.to_categorical(test_y, 10)

# create sequential models
model = Sequential()

# The first convolutional layer: 6 convolution kernels, the size is 5*5, relu activation function
model.add(Conv2D(6, kernel_size = (5,5), activation='relu', input_shape=(28, 28, 1)))

# the second pooling layer: maximum pooling
model.add(MaxPooling2D(pool_size = (2, 2)))

# the third convolutional layer: 16 convolution kernels, the size is 5*5, relu activation function
model.add(Conv2D(16, kernel_size = (5, 5), activation = 'relu'))

# the second pooling layer: maximum pooling
model.add(MaxPooling2D(pool_size = (2, 2)))

# Flatten the parameters, which is called a convolutional layer in leNet5. in fact, this layer is a one-dimensional vector, the same as the fully connected layer
model.add(Flatten())
model.add(Dense(120, activation = 'relu'))

# Fully connected layer, the number of output nodes is 84
model.add(Dense(84, activation = 'relu'))

# The output layer uses the softmax activation function to calculate the classification probability
model.add(Dense(10, activation='softmax'))

# set the loss function and optimizer configuration
model.compile(loss = keras.metrics.categorical_crossentropy, optimizer = keras.optimizers.Adam(), metrics = ['accuracy'])

# Incoming training data for training
model.fit(train_x, train_y, batch_size = 128, epochs = 2, verbose = 1, validation_data = (test_x, test_y))

# Evaluate the results
score = model.evaluate(test_x, test_y)
print('Error: %.4lf' % score[0])
print('Accuracy: ', score[1])