昨天日常睡前刷B站，看到一个很糟心的话题

确实，抑郁很不好受。深夜常常一个人翻来覆去睡不着，眼泪不受控制的往下流，仿佛被世界抛弃，又或是突然的情绪爆发……经历很长一段时间后我走出了这个阴影，但是对某些人我可能会愧疚一辈子，我只能默默关注着她发的微博，但是不能去给她任何希望，所以每次看着她这么痛苦都会深深自责



负面情绪完全是她生活的主旋律，然后还看到其他类似的人

所以我就去到处找相关的项目，希望能用我们学的知识去帮助这些需要我们关心的人。
走饭微博的评论网址
去看这下面的评论，可以看到很多这样对生活很不乐观的人，不想让悲剧发生，所以我们要爬取所有评论，设计算法找到那些有放弃自己倾向的人

代码

然后半夜的时候我找了相关的各种链接，然后在一个公众号找到了相关的内容，具体的项目在GitHub，大家可以去这里查看。
具体的代码我也贴在这里
爬虫部分

import json
import requests
import time
import os
from lxml import etree

"""
爬取微博评论
"""

headers = {
    
    'Referer': 'https://weibo.com/1648007681/yark9qWbM?type=comment',
    'X-Requested-With': 'XMLHttpRequest',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36',
    'Cookie': '填自己的cookie',
}


# 下载图片
def download_pic(url, nick_name):
    pic_file_path = os.path.join(os.path.abspath(''), 'pic')
    # 当前路径+pic
    if not url:
        return
    if not os.path.exists(pic_file_path):
        os.mkdir(pic_file_path)
    resp = requests.get(url)
    if resp.status_code == 200:
        with open(pic_file_path + f'/{nick_name}.jpg', 'wb') as f:
            f.write(resp.content)
    time.sleep(2)

# 写入留言内容
def write_comment(comment, pic_url, nick_name):
    f = open('comment.txt', 'a', encoding='utf-8')
    for index, i in enumerate(comment):
        if ':' not in i and '回复' not in i and i != '':
            # 去除评论的评论
            w_comment = "".join(i.split('：')[1:])
            print(w_comment)
            w_comment = i.strip().replace('\n', '')
            # 写入评论
            f.write(w_comment.replace('等人', '').replace('图片评论', '')+'\n')
            # 获得头像
            download_pic(pic_url[index], nick_name[index])

if __name__ == '__main__':

    params = {
    
        'ajwvr': 6,
        'id': '3424883176420210',
        'page': 1,
        '_rnd': int(round(time.time() * 1000))
    }
    URL = 'https://weibo.com/aj/v6/comment/big'

    for num in range(1,25,1):
        print(f'====== 正在读取第 {num} 页 ========')
        params['page'] = num
        params['_rnd'] = int(round(time.time() * 1000))
        print(params['_rnd'])
        resp = requests.get(URL, params=params, headers=headers)
        resp = json.loads(resp.text)
        if resp['code'] == '100000':
            html = resp['data']['html']
            html = etree.HTML(html)
            data = html.xpath('//div[@node-type="comment_list"]')
            for i in data:
                # 评论人昵称
                nick_name = i.xpath('.//div[@class="WB_text"]/a[1]/text()')
                # 评论内容
                text = i.xpath('.//div[@class="WB_text"]')
                text = [i.xpath('string(.)') for i in text]
                # 头像地址
                pic_url = i.xpath('.//div[@class="WB_face W_fl"]/a/img/@src')

                print(len(nick_name),len(text),len(pic_url))
                # 写入文件并下载头像
                write_comment([i.strip() for i in text], pic_url, nick_name)
        time.sleep(5)

具体实现方法就是去找网页，很简单就可以找到评论的json页


请求再解析网页，得到数据存入txt就行

分析判断部分

import sys
import numpy as np
import segment
from sklearn.model_selection import cross_val_score
from sklearn.metrics import classification_report
from sklearn import metrics
from sklearn.externals import joblib

seg = segment.segment()
print('(1) load texts...')

pos_train_filename = r'data\source\normal.txt'
neg_train_filename = r'data\source\die.txt'
pos_eval_filename = r'data\source\normal_test.txt'
neg_eval_filename = r'data\source\die_test.txt'

origin_pos_train = open(pos_train_filename, encoding='UTF-8').read().split('\n')
origin_neg_train = open(neg_train_filename, encoding='UTF-8').read().split('\n')
origin_pos_eval = open(pos_eval_filename, encoding='UTF-8').read().split('\n')
origin_neg_eval = open(neg_eval_filename, encoding='UTF-8').read().split('\n')

pos_train_dir, pos_train_label_dir = seg.seg_lines_list(1, pos_train_filename)
neg_train_dir, neg_train_label_dir = seg.seg_lines_list(0, neg_train_filename)
pos_test_dir, pos_test_label_dir = seg.seg_lines_list(1, pos_eval_filename)
neg_test_dir, neg_test_label_dir = seg.seg_lines_list(0, neg_eval_filename)

train_pos = open(pos_train_dir, encoding='UTF-8').read().split('\n')
train_neg = open(neg_train_dir, encoding='UTF-8').read().split('\n')
test_pos = open(pos_test_dir, encoding='UTF-8').read().split('\n')
test_neg = open(neg_test_dir, encoding='UTF-8').read().split('\n')

train_pos_label = open(pos_train_label_dir, encoding='UTF-8').read().split('\n')
train_neg_label = open(neg_train_label_dir, encoding='UTF-8').read().split('\n')
test_pos_label = open(pos_test_label_dir, encoding='UTF-8').read().split('\n')
test_neg_label = open(neg_test_label_dir, encoding='UTF-8').read().split('\n')

origin_train_text = origin_pos_train + origin_neg_train
origin_eval_text = origin_pos_eval + origin_neg_eval
train_texts = train_pos + train_neg
test_texts = test_pos + test_neg
train_labels = train_pos_label + train_neg_label
test_labels = test_pos_label + test_neg_label

all_text = train_texts + test_texts
all_labels = train_labels + test_labels

print('(2) doc to var...')
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer

# CountVectorizer考虑每种词汇在该训练文本中出现的频率，得到计数矩阵
count_v0= CountVectorizer(analyzer='word',token_pattern='\w{1,}')
counts_all = count_v0.fit_transform(all_text)

count_v1= CountVectorizer(vocabulary=count_v0.vocabulary_)
counts_train = count_v1.fit_transform(train_texts) 
print("the shape of train is "+repr(counts_train.shape)  )
count_v2 = CountVectorizer(vocabulary=count_v0.vocabulary_)
counts_test = count_v2.fit_transform(test_texts)
print("the shape of test is "+repr(counts_test.shape)  )

# 保存数字化后的词典
joblib.dump(count_v0.vocabulary_, "model/die_svm_20191110_vocab.m")

counts_all = count_v2.fit_transform(all_text)
print("the shape of all is "+repr(counts_all.shape))

# 将计数矩阵转换为规格化的tf-idf格式
tfidftransformer = TfidfTransformer()  
train_data = tfidftransformer.fit(counts_train).transform(counts_train)
test_data = tfidftransformer.fit(counts_test).transform(counts_test)
all_data = tfidftransformer.fit(counts_all).transform(counts_all)

train_data = counts_train
test_data = counts_test
all_data = counts_all

x_train = train_data
y_train = train_labels
x_test = test_data
y_test = test_labels

print('(3) SVM...')
from sklearn.svm import SVC

# 使用线性核函数的SVM分类器，并启用概率估计（分别显示分到两个类别的概率如：[0.12983359 0.87016641]）
svclf = SVC(kernel = 'linear', probability=True) 

# 开始训练
svclf.fit(x_train,y_train)
# 保存模型
joblib.dump(svclf, "model/die_svm_20191110.m")

# 测试集进行测试
preds = svclf.predict(x_test)
y_preds = svclf.predict_proba(x_test)

preds = preds.tolist()
for i,pred in enumerate(preds):
    # 显示被分错的微博
    if int(pred) != int(y_test[i]):
        try:
            print(origin_eval_text[i], ':', test_texts[i], pred, y_test[i], y_preds[i])
        except Exception as e:
            print(e)

# 分别查看两个类别的准确率、召回率和F1值
print(classification_report(y_test, preds))

下面是segment.py

import jieba
import jieba.posseg as psg
import os
import math
import re
from utils import files_processing
 
'''
read() 每次读取整个文件，它通常将读取到底文件内容放到一个字符串变量中，也就是说 .read() 生成文件内容是一个字符串类型。
readline()每只读取文件的一行，通常也是读取到的一行内容放到一个字符串变量中，返回str类型。
readlines()每次按行读取整个文件内容，将读取到的内容放到一个列表中，返回list类型。
'''

class segment():

    user_path = 'data/n.txt'
    jieba.load_userdict(user_path)
 
    stopwords_path='data/stopwords.txt'
    stopwords = []
    with open(stopwords_path, "r", encoding='utf8') as f:
        lines = f.readlines()
        for line in lines:
            stopwords.append(line.strip())
    
    def segment_line(self, file_list,segment_out_dir,stopwords=[]):
        '''
        字词分割，对每行进行字词分割
        :param file_list:
        :param segment_out_dir:
        :param stopwords:
        :return:
        '''
        for i,file in enumerate(file_list):
            segment_out_name=os.path.join(segment_out_dir,'segment_{}.txt'.format(i))
            segment_file = open(segment_out_name, 'a', encoding='utf8')
            with open(file, encoding='utf8') as f:
                text = f.readlines()
                for sentence in text:
                    # jieba.cut():参数sentence必须是str(unicode)类型
                    sentence = list(jieba.cut(sentence))
                    sentence_segment = []
                    for word in sentence:
                        if word not in stopwords:
                            sentence_segment.append(word)
                    segment_file.write(" ".join(sentence_segment))
                del text
                f.close()
            segment_file.close()

    def segment_lines(self, file_list,segment_out_dir,stopwords=[]):
        '''
        字词分割，对整个文件内容进行字词分割
        :param file_list:
        :param segment_out_dir:
        :param stopwords:
        :return:
        '''

        for i,file in enumerate(file_list):
            segment_out_name=os.path.join(segment_out_dir,str(file.split('\\')[1].split('.')[0])+'_segment.txt')

            with open(file, 'r', encoding='utf-8') as f1, open(segment_out_name, 'w', encoding='utf-8') as f2:
                for line in f1.readlines():
                    sentence_segment = []
                    property = []
                    document_cut = psg.cut(line.strip())
                    for x in document_cut:
                        if x.word not in stopwords:
                            sentence_segment.append(x.word)
                            property.append(x.flag)
                    # result = ' '.join(sentence_segment) + ' ' + ' '.join(property) + '\n'
                    result = ' '.join(sentence_segment) + '\n'
                    f2.write(result)
    def split_lines_list(self, type, file, stopwords=stopwords):
        '''
        字词分割，对整个文件内容进行字词分割
        :param file:
        :param stopwords:
        :return:
        '''
        file_ = open(file, 'r', encoding='utf-8').readlines()
        name = str(file.split('\\')[-1].split('.')[0])

        segment_out_name=os.path.join('./data/segment',name+'_segment.txt')
        label_out_name=os.path.join('./data/segment',name+'_label.txt')

        file_segment = open(segment_out_name, 'w', encoding='utf-8')
        label = open(label_out_name, 'w', encoding='utf-8')

        for index, i in enumerate(file_):
            sentence_segment = []
            # property = []
            p = re.compile('(\\{..)|(\[.*\])|([\uD800-\uDBFF])|([\uDC00-\uDFFF])')
            i=p.sub( '', i.strip())
            # document_cut = jieba.cut(i.strip(), cut_all=False)
            # for x in document_cut:
            #     if x not in stopwords:
            #         sentence_segment.append(x)
                    # property.append(x.flag)
            # result = ' '.join(sentence_segment) + ' ' + ' '.join(property) + '\n'
            if index < len(file_)-1:
                result = ' '.join(i) + '\n'
                label.write(str(type) + '\n')
            else:
                result = ' '.join(i) 
                label.write(str(type))
            
            file_segment.write(result)
        file_segment.close()
        label.close()

        return segment_out_name, label_out_name
        
    def seg_lines_list(self, type, file, stopwords=stopwords):
        '''
        字词分割，对整个文件内容进行字词分割
        :param file:
        :param stopwords:
        :return:
        '''
        file_ = open(file, 'r', encoding='utf-8').readlines()
        name = str(file.split('\\')[-1].split('.')[0])

        segment_out_name=os.path.join('./data/segment',name+'_segment.txt')
        label_out_name=os.path.join('./data/segment',name+'_label.txt')

        file_segment = open(segment_out_name, 'w', encoding='utf-8')
        label = open(label_out_name, 'w', encoding='utf-8')

        for index, i in enumerate(file_):
            sentence_segment = []
            # property = []
            p = re.compile('(/{..)|(\[.*\])|([\uD800-\uDBFF][\uDC00-\uDFFF][\U00010000-\U0010ffff])')
            i=p.sub( '', i.strip())
            document_cut = jieba.cut(i.strip(), cut_all=False)
            for x in document_cut:
                if x not in stopwords:
                    sentence_segment.append(x)
                    # property.append(x.flag)
            # result = ' '.join(sentence_segment) + ' ' + ' '.join(property) + '\n'
            if index < len(file_)-1:
                result = ' '.join(sentence_segment) + '\n'
                label.write(str(type) + '\n')
            else:
                result = ' '.join(sentence_segment) 
                label.write(str(type))
            
            file_segment.write(result)
        file_segment.close()
        label.close()

        return segment_out_name, label_out_name

    def MergeTxt(filepath,outfile):
        k = open(filepath+outfile, 'a+', encoding='utf-8')
        for parent, dirnames, filenames in os.walk(filepath):
            for filepath in filenames:
                txtPath = os.path.join(parent, filepath) 
                f = open(txtPath, encoding='utf-8')
                k.write(f.read()+"\n")

if __name__=='__main__':
    # 多线程分词
    # jieba.enable_parallel()
    # 加载自定义词典
    user_path = 'data/n.txt'
    jieba.load_userdict(user_path)
 
    stopwords_path='data/stopwords.txt'
    stopwords=getStopwords(stopwords_path)
 
    file_dir='data/source/biaozhu'
    segment_out_dir='data/segment/biaozhu_property'
    file_list=files_processing.get_files_list(file_dir,postfix='*.txt')
    segment_lines(file_list, segment_out_dir, stopwords)
    # segment_lines(file_list, segment_out_dir)

这部分代码很多，大家还是去GitHub下载吧，就不多贴了。
看看运行的结果

虽然有很多负面情绪的评论没有识别出来，但是可以看到准确率还是比较好的

最后

希望大家都去关注一下这个项目，模型主要是svm，后期我想利用现有的数据自己动手写个lstm看看能不能提高分类的准确率。身边需要我们关怀的人很多很多，从亲人，到朋友，最后到陌生人，当他们不顺的时候也许你的一句问候就是他们重新振作的力量。