{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "b'{\"a_alarmmaxalarmlevel\": 0, \"dt\": 20211201, \"electstoraempdataentitylist\": \"[3*-2*1638357832000*1*16*8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]\", \"electstoragevoltdataentitylist\": \"[3*-2*1638357832000*1*358.9*0.4*96*1*96*3.741,3.739,3.739,3.739,3.739,3.739,3.736,3.739,3.738,3.739,3.739,3.739,3.738,3.739,3.738,3.739,3.736,3.739,3.738,3.741,3.739,3.739,3.742,3.741,3.742,3.742,3.744,3.741,3.736,3.741,3.739,3.738,3.741,3.738,3.736,3.742,3.736,3.723,3.738,3.738,3.738,3.739,3.742,3.739,3.736,3.738,3.742,3.741,3.738,3.736,3.741,3.738,3.736,3.738,3.738,3.738,3.736,3.739,3.741,3.742,3.738,3.739,3.739,3.739,3.738,3.738,3.738,3.739,3.738,3.741,3.733,3.739,3.738,3.739,3.738,3.736,3.738,3.736,3.738,3.741,3.738,3.739,3.738,3.739,3.733,3.739,3.738,3.742,3.738,3.736,3.744,3.739,3.738,3.739,3.736,3.739]\", \"ep_faultcodes\": NaN, \"ex_maxbattempassemblyno\": 1, \"ex_maxbattempprobeinassemblycode\": 1, \"ex_maxvoltbatmonomercode\": 27, \"ex_minbattempprobeinassemblycode\": 1, \"ex_minbattempprobeno\": 1, \"ex_minvoltbatmonomercode\": 38, \"g_acceleratorpedalstroke\": 0, \"g_chargestatus\": 3, \"g_mileage\": 50717.0, \"g_nsulationresistance\": 23932, \"g_soc\": 60, \"g_speed\": 0.0, \"g_status\": 2, \"g_totalcurrent\": 0.4, \"g_totalvoltage\": 358.0, \"l_altitude\": 0.0, \"l_lat\": 39.548158, \"l_lng\": 116.688966, \"model\": \"S00\", \"motordataentitylist\": \"[3*-2*1638357832000*1*1*30*-1*-0.9*39*236.7*0.0]\", \"p_bmsbatterytotalcapacity\": NaN, \"p_bmschargeccsig\": NaN, \"p_bmschargecpsig\": NaN, \"p_bmschargingallowed\": NaN, \"p_bmschargingstationmaxoutpa\": NaN, \"p_bmschargingstationmaxoutpv\": NaN, \"p_bmschargingstationminoutpa\": NaN, \"p_bmschargingstationminoutpv\": NaN, \"p_bmsclmthermalsts\": NaN, \"p_bmsdcchargecontsts\": NaN, \"p_bmsdiffbatttemp\": NaN, \"p_bmsdiffcellvoltage\": NaN, \"p_bmsfastchargecc\": NaN, \"p_bmsfastcharotaltime\": NaN, \"p_bmsfccurrentrequest\": NaN, \"p_bmsfctype\": NaN, \"p_bmsfcvoltagerequest\": NaN, \"p_bmshdsahactivecpsum\": NaN, \"p_bmshighestbatttemp\": NaN, \"p_bmshighestcellvoltage\": NaN, \"p_bmslowestbatttemp\": NaN, \"p_bmslowestcellvoltage\": NaN, \"p_bmsnegbattcontsts\": NaN, \"p_bmsnegfastchgporttem\": NaN, \"p_bmsnegslowchgporttem\": NaN, \"p_bmsposbattcontsts\": NaN, \"p_bmsposfastchgporttem\": NaN, \"p_bmsposslowchgporttem\": NaN, \"p_bmsprechargecontsts\": NaN, \"p_bmsquichgcc\": NaN, \"p_bmsquichgpileoutputcur\": NaN, \"p_bmsquichgpileoutputvol\": NaN, \"p_bmsrequestoutputcurrent\": NaN, \"p_bmsrequestoutputmode\": NaN, \"p_bmsrequestoutputvoltage\": NaN, \"p_bmss2switch\": NaN, \"p_bmsslowchargecc\": NaN, \"p_bmsslowcharotaltim\": NaN, \"p_bmsslowchgcp\": NaN, \"p_bmsworkingmode\": NaN, \"p_clmacsts\": NaN, \"p_obcchargecurrent\": NaN, \"p_obcchargevoltage\": NaN, \"p_obccmstate\": NaN, \"p_obcs2sts\": NaN, \"p_ptcpowerconsumption\": NaN, \"p_ptcwrkngsts\": NaN, \"p_vcuhvilmajorloop\": NaN, \"p_vcupmmpwrdwnst\": NaN, \"p_vcupmmpwrupst\": NaN, \"time_value\": \"2021-11-30 16:27:07\", \"v_airouttemp\": 6.0, \"v_airstatus\": 0, \"v_avgpowerconsumption\": NaN, \"v_drivemode\": NaN, \"v_instpowerconsumption\": NaN, \"v_poweralreadytime\": 0, \"v_powerchargestatus\": 0, \"v_powerresiduemileage\": 174.7, \"v_subpowerconsumption\": NaN, \"v_tripmeter\": NaN, \"v_wheelangle\": NaN, \"v_xaxisacceleration\": NaN, \"v_yaxisacceleration\": NaN, \"vin\": \"LUZAGAAA5JA000974\"}'\n" ] } ], "source": [ "import ssl\n", "from kafka import KafkaProducer, KafkaConsumer\n", "from kafka.errors import kafka_errors\n", "import traceback\n", "import json\n", "import pdb\n", "import os\n", "import pandas as pd\n", "import oss2\n", "from itertools import islice\n", "import pandas as pd\n", "context = ssl.create_default_context()\n", "context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)\n", "## The new python(2.7.8+) may cannot ignore the hostname check,\n", "## you could set to ssl.CERT_NONE to walk around the problem,\n", "## or you can change the client to confluent-python-demo \n", "\n", "# context.verify_mode = ssl.CERT_NONE\n", "context.verify_mode = ssl.CERT_REQUIRED\n", "topic = 'test' # toAlg, toAlgSn\n", "# context.check_hostname = True\n", "context.load_verify_locations(\"ca-cert\")\n", "consumer = KafkaConsumer(topic, bootstrap_servers=['alikafka-pre-cn-8ed2kw57901x-1.alikafka.aliyuncs.com:9093',\n", " 'alikafka-pre-cn-8ed2kw57901x-2.alikafka.aliyuncs.com:9093',\n", " 'alikafka-pre-cn-8ed2kw57901x-3.alikafka.aliyuncs.com:9093'],\n", " sasl_mechanism=\"PLAIN\",\n", " ssl_context=context,\n", " security_protocol='SASL_SSL',\n", " api_version = (2,2),\n", " sasl_plain_username='alikafka_pre-cn-8ed2kw57901x',\n", " sasl_plain_password='hlQsApgUUdUxcWEr1uQoM9BeuF8t8vMF',\n", " consumer_timeout_ms= 100000000, # 如果10秒内kafka中没有可供消费的数据，自动退出\n", " client_id='consumer-python3',\n", " auto_offset_reset='earliest',# 消费kafka中最近的数据，如果设置为earliest则消费最早的数据，不管这些数据是否消费\n", " enable_auto_commit=True, # 自动提交消费者的offset\n", " auto_commit_interval_ms=3000, ## 自动提交消费者offset的时间间隔,\n", "\n", ")\n", "\n", "columns = [\"id\",\"VIN\",\"VehModel\",\"VehState\",\"Time\",\"CellVoltage\",\"PackVoltage\",'CellMaxVolNum',\"CellMinVolNum\",'CellTemp','CellMaxTempNum','CellMinTempNum','PackCrnt','VehRmnRng','VehOdo','FltCode',\n", "'FltLvl','VehSpd','PedalAngle','PackSoc','MotorPwr','EnmTemp','InsulationRss','ChrgSta','GPS_lon','GPS_lat','GPS_alt',\n", " 'PowerAlreadyTime','BMSSta','ACSta','OncCmState','HVILSta','OfcAccumTime','OncAccumTime','BmsOncChgCp','BmsOfcPileOutputVol','BmsOfcPileOutputCur','BmsOfcCc','BmsOfcAllowed',\n", " 'OncOutCurrent','OncOutVoltage','OfcStationMaxOutVolt','OfcStationMinOutVolt','OfcStationMaxOutCrnt','OfcStationMinOutCrnt','BmsOfcgVoltageRequest','BmsOfcType','BmsOfcCurrentRequest',\n", " 'BmsOncChgRequest','BmsRequestOutputCurrent','BmsRequestOutputVoltage','BmsOncCc','BmsS2State','HvUpState','HvDownState','CellMaxVol','CellMinVol','CellMaxTemp','CellMinTemp','Tripmeter',\n", " 'VehAccX','VehAccY','StringAngle','PTCSta','PTCPwr','PosRlySta','NegRlySta','OfcPreRlySta','OfcRlySta','OfcPosChrgerTemp','OfcNegChrgerTemp','OncPosChrgerTemp','OncNegChrgerTemp',\n", " 'InstPwrCmsmtion','AvgPwrCmsmtion','SubPwrCmsmtion','BmsCellVoltDiff','DriveMode','BmsBatTempDiff','BmsClmThermalSts','BmsBatteryTotalCapacity','BmsBatteryAvaiCapacity',\n", " 'BmsOncChgCpConn','CellVoltRecTime','CellVoltTotalCount','CellVoltFrameNo','CellVoltFrameCount','CellTempTotalCount','SN','BatteryModel']\n", "auth = oss2.Auth('LTAI4FyoDnXMjqd78rdRDFHN', 'sdFl6vjM9l2rvWiUTzuFQb2xqjUoY5')\n", "bucket = oss2.Bucket(auth, \"https://oss-cn-hangzhou.aliyuncs.com\", 'ff-hezhong')\n", " \n", "\n", "for message in consumer:\n", " print(message.value)\n", "\n", " \n", " \n", "\n" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "df = pd.read_csv('s_kafka_tbox_analysis_hh_delta_ext_topic.csv')" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "data_dict = df.to_dict(orient='records')" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "df = pd.read_csv(\"s_kafka_tbox_analysis_hh_delta_ext_topic.csv\")" ] }, { "cell_type": "code", "execution_count": 33, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0 2021-11-30 16:27:17\n", "1 2021-12-01 19:18:08\n", "2 2021-12-01 19:36:38\n", "3 2021-12-01 19:01:38\n", "4 2021-12-01 19:41:20\n", " ... \n", "14894 2021-12-01 00:03:18\n", "14895 2021-12-01 00:50:21\n", "14896 2021-12-01 00:51:01\n", "14897 2021-12-01 00:34:04\n", "14898 2021-12-01 00:03:50\n", "Name: time_value, Length: 14899, dtype: object" ] }, "execution_count": 33, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df['time_value']" ] }, { "cell_type": "code", "execution_count": 36, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "4373 2021-11-30 16:27:07\n", "13024 2021-11-30 16:27:07\n", "0 2021-11-30 16:27:17\n", "10539 2021-11-30 16:27:17\n", "5563 2021-11-30 16:27:27\n", " ... \n", "12512 2021-12-02 00:05:17\n", "11892 2021-12-02 00:05:27\n", "2791 2021-12-02 00:52:22\n", "884 2021-12-02 00:52:32\n", "9588 2021-12-02 00:52:42\n", "Name: time_value, Length: 14899, dtype: object" ] }, "execution_count": 36, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.sort_values(['time_value'])['time_value']" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "interpreter": { "hash": "5ac93a79b26608c768d42fbd754dd4f69161017bfc4d38cb4b18d60e5198dbca" }, "kernelspec": { "display_name": "Python 3.6.2 ('ana_py36')", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.2" }, "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2 }