在 pyspark 中读取 DStrem 中的嵌套 JSON 数据
Read Nested JSON Data in DStrem in pyspark
我编写了以下代码来从 Tweepy API 流式传输数据。我正在获取流对象中的数据。但是获取不到streamp["user"]["followers_count"]但是不知道怎么获取。我也试过 jsonLines = lines.flatMap(lambda json_str:json.loads(json_str)) 但没有帮助。
from __future__ import print_function
from pyspark import SparkContext
from pyspark.streaming import StreamingContext
import json
sc = SparkContext()
ssc = StreamingContext(sc, 10) #Slide Interval of 10 sec
socket_stream = ssc.socketTextStream("localhost",4444)
stream = socket_stream.window(30) #window length 30 sec
stream.pprint()
ssc.start()
ssc.awaitTermination()
stream.pprint() 给了我以下 JSON.
{"created_at":"Sat May 08 09:27:43 +0000 2021","id":1390961604079067137,"id_str":"1390961604079067137","text":"The return of the king! This style has so many parameters that have to be correct and Stone nails em all! - Drinkin\u2026 https:\/\/t.co\/uCogsfW8NC","source":"\u003ca href=\"https:\/\/untappd.com\" rel=\"nofollow\"\u003eUntappd\u003c\/a\u003e","truncated":true,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":905439001,"id_str":"905439001","name":"Nahoj Morts","screen_name":"CraftBeerJunkie","location":null,"url":null,"description":null,"translator_type":"none","protected":false,"verified":false,"followers_count":15,"friends_count":113,"listed_count":0,"favourites_count":1,"statuses_count":1868,"created_at":"Fri Oct 26 06:36:01 +0000 2012","utc_offset":null,"time_zone":null,"geo_enabled":false,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/2765713047\/b92d58c569ad4739e67ef4d4e9a35780_normal.jpeg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/2765713047\/b92d58c569ad4739e67ef4d4e9a35780_normal.jpeg","default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null,"withheld_in_countries":[]},"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"extended_tweet":{"full_text":"The return of the king! This style has so many parameters that have to be correct and Stone nails em all! - Drinking a Stone Sublimely Self-Righteous Black IPA by @StoneBrewing @ Uggleberget \u2014 https:\/\/t.co\/a3XixD5teo","display_text_range":[0,217],"entities":{"hashtags":[],"urls":[{"url":"https:\/\/t.co\/a3XixD5teo","expanded_url":"https:\/\/untp.beer\/s\/c1025322433","display_url":"untp.beer\/s\/c1025322433","indices":[194,217]}],"user_mentions":[{"screen_name":"StoneBrewing","name":"Stone Brewing","id":16331259,"id_str":"16331259","indices":[163,176]}],"symbols":[]}},"quote_count":0,"reply_count":0,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"urls":[{"url":"https:\/\/t.co\/uCogsfW8NC","expanded_url":"https:\/\/twitter.com\/i\/web\/status\/1390961604079067137","display_url":"twitter.com\/i\/web\/status\/1\u2026","indices":[117,140]}],"user_mentions":[],"symbols":[]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"low","lang":"en","timestamp_ms":"1620466063886"}
{"created_at":"Sat May 08 09:27:43 +0000 2021","id":1390961604334919683,"id_str":"1390961604334919683","text":"[HQ] 210508 #KrisWu #Wuyifan @ GTSSC 2021 (Day 1) Cr.Fanbaobao(3) #PorscheRacerKrisWu \nhttps:\/\/t.co\/cEcAtzKNsM\u2026 https:\/\/t.co\/3B9uUS3mdu","display_text_range":[0,140],"source":"\u003ca href=\"https:\/\/mobile.twitter.com\" rel=\"nofollow\"\u003eTwitter Web App\u003c\/a\u003e","truncated":true,"in_reply_to_status_id":1390961304316387331,"in_reply_to_status_id_str":"1390961304316387331","in_reply_to_user_id":620222301,"in_reply_to_user_id_str":"620222301","in_reply_to_screen_name":"kissmemyfan_","user":{"id":620222301,"id_str":"620222301","name":"KISSMEMYFAN1106\u2661","screen_name":"kissmemyfan_","location":null,"url":"https:\/\/www.youtube.com\/channel\/UCtYrDyYwqZyQfYtCz79zPHQ\/featured","description":"for @KrisWu Kris Wu Yifan since 120627 \u2740notify in case of emergency,pls dm\u2740 fb: https:\/\/www.facebook.com\/groups\/1051879991503131\/","translator_type":"regular","protected":false,"verified":false,"followers_count":91307,"friends_count":24,"listed_count":433,"favourites_count":14194,"statuses_count":95476,"created_at":"Wed Jun 27 16:34:02 +0000 2012","utc_offset":null,"time_zone":null,"geo_enabled":true,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"FFCC4D","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":true,"profile_link_color":"E81C4F","profile_sidebar_border_color":"FFFFFF","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":false,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/1252957871802707974\/RjNGNnly_normal.jpg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/1252957871802707974\/RjNGNnly_normal.jpg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/620222301\/1587563426","default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null,"withheld_in_countries":[]},"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"extended_tweet":{"full_text":"[HQ] 210508 #KrisWu #Wuyifan @ GTSSC 2021 (Day 1) Cr.Fanbaobao(3) #PorscheRacerKrisWu \nhttps:\/\/t.co\/cEcAtzKNsM\nhttps:\/\/t.co\/PNbPqE88Y3 https:\/\/t.co\/8WPhfpHiyO","display_text_range":[0,134],"entities":{"hashtags":[{"text":"KrisWu","indices":[12,19]},{"text":"Wuyifan","indices":[20,28]},{"text":"PorscheRacerKrisWu","indices":[66,85]}],"urls":[{"url":"https:\/\/t.co\/cEcAtzKNsM","expanded_url":"https:\/\/wx1.sinaimg.cn\/large\/00688bwvly1gqb3c1cdbuj31l82bpe81.jpg","display_url":"wx1.sinaimg.cn\/large\/00688bwv\u2026","indices":[87,110]},{"url":"https:\/\/t.co\/PNbPqE88Y3","expanded_url":"https:\/\/wx4.sinaimg.cn\/large\/00688bwvly1gqb3c1mc9hj31ay1wb7ji.jpg","display_url":"wx4.sinaimg.cn\/large\/00688bwv\u2026","indices":[111,134]}],"user_mentions":[],"symbols":[],"media":[{"id":1390961423400857602,"id_str":"1390961423400857602","indices":[135,158],"media_url":"http:\/\/pbs.twimg.com\/media\/E02wC4IUcAI4F-O.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E02wC4IUcAI4F-O.jpg","url":"https:\/\/t.co\/8WPhfpHiyO","display_url":"pic.twitter.com\/8WPhfpHiyO","expanded_url":"https:\/\/twitter.com\/kissmemyfan_\/status\/1390961604334919683\/photo\/1","type":"photo","sizes":{"small":{"w":465,"h":680,"resize":"fit"},"thumb":{"w":150,"h":150,"resize":"crop"},"large":{"w":1024,"h":1498,"resize":"fit"},"medium":{"w":820,"h":1200,"resize":"fit"}}},{"id":1390961423417704450,"id_str":"1390961423417704450","indices":[135,158],"media_url":"http:\/\/pbs.twimg.com\/media\/E02wC4MVgAI8ZHZ.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E02wC4MVgAI8ZHZ.jpg","url":"https:\/\/t.co\/8WPhfpHiyO","display_url":"pic.twitter.com\/8WPhfpHiyO","expanded_url":"https:\/\/twitter.com\/kissmemyfan_\/status\/1390961604334919683\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":825,"h":1200,"resize":"fit"},"small":{"w":467,"h":680,"resize":"fit"},"large":{"w":1024,"h":1490,"resize":"fit"}}}]},"extended_entities":{"media":[{"id":1390961423400857602,"id_str":"1390961423400857602","indices":[135,158],"media_url":"http:\/\/pbs.twimg.com\/media\/E02wC4IUcAI4F-O.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E02wC4IUcAI4F-O.jpg","url":"https:\/\/t.co\/8WPhfpHiyO","display_url":"pic.twitter.com\/8WPhfpHiyO","expanded_url":"https:\/\/twitter.com\/kissmemyfan_\/status\/1390961604334919683\/photo\/1","type":"photo","sizes":{"small":{"w":465,"h":680,"resize":"fit"},"thumb":{"w":150,"h":150,"resize":"crop"},"large":{"w":1024,"h":1498,"resize":"fit"},"medium":{"w":820,"h":1200,"resize":"fit"}}},{"id":1390961423417704450,"id_str":"1390961423417704450","indices":[135,158],"media_url":"http:\/\/pbs.twimg.com\/media\/E02wC4MVgAI8ZHZ.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E02wC4MVgAI8ZHZ.jpg","url":"https:\/\/t.co\/8WPhfpHiyO","display_url":"pic.twitter.com\/8WPhfpHiyO","expanded_url":"https:\/\/twitter.com\/kissmemyfan_\/status\/1390961604334919683\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":825,"h":1200,"resize":"fit"},"small":{"w":467,"h":680,"resize":"fit"},"large":{"w":1024,"h":1490,"resize":"fit"}}}]}},"quote_count":0,"reply_count":0,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[{"text":"KrisWu","indices":[12,19]},{"text":"Wuyifan","indices":[20,28]},{"text":"PorscheRacerKrisWu","indices":[66,85]}],"urls":[{"url":"https:\/\/t.co\/cEcAtzKNsM","expanded_url":"https:\/\/wx1.sinaimg.cn\/large\/00688bwvly1gqb3c1cdbuj31l82bpe81.jpg","display_url":"wx1.sinaimg.cn\/large\/00688bwv\u2026","indices":[87,110]},{"url":"https:\/\/t.co\/3B9uUS3mdu","expanded_url":"https:\/\/twitter.com\/i\/web\/status\/1390961604334919683","display_url":"twitter.com\/i\/web\/status\/1\u2026","indices":[112,135]}],"user_mentions":[],"symbols":[]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"low","lang":"ht","timestamp_ms":"1620466063947"}
{"created_at":"Sat May 08 09:27:44 +0000 2021","id":1390961607572865031,"id_str":"1390961607572865031","text":"RT @aa86marat2: #\u0644\u0627_\u0644\u0644\u062a\u0637\u0639\u064a\u0645_\u0627\u0644\u0627\u062c\u0628\u0627\u0631\u064a\n\u0628\u0633\u0645 \u0627\u0644\u0644\u0647 \u062a\u0648\u0643\u0644\u062a \u0639\u0644\u0649 \u0627\u0644\u0644\u0647 \n\u0648\u0627\u0644\u0644\u0647 \u0643\u0631\u064a\u0645 \u064a\u0627\u0631\u0628 \u062a\u062a\u0642\u0641\u0644 \u0627\u0644\u064a\u0648\u0645 \u0628\u0648\u062c\u0648\u062f\u0627\u0644\u062e\u064a\u0631\u064a\u0646 \u0627\u0645\u062b\u0627\u0644\u0643\u0645\n \u064a\u0627\u0631\u0628 \u0633\u062e\u0631\u0644\u0647 \u0645\u0646 \u064a\u0642\u0641\u0644\u0647\u0627\n\u0633\u062c\u064a\u0646\u2026","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":1250758860660527105,"id_str":"1250758860660527105","name":"om_mhamd99","screen_name":"Mhamd99Om","location":null,"url":null,"description":"\u064a\u0627\u0631\u0628 \u2728\ud83e\udd32\ud83c\udffb\u0628\u0627\u0631\u0643 \u0644\u064a \u0641\u064a\u0645\u0627 \u0623\u0639\u0637\u062a\u0646\u064a \u0648\u0641\u0631\u062c \u0647\u0645\u064a \u0628\u0642\u0636\u0627\u0621 \u062f\u064a\u0646 \u0648\u0627\u0639\u062a\u064a \u0639\u0644\u064a \u0630\u0644\u0643 \u0641\u0627\u062a\u0648\u0631\u0647 \u062a\u0646\u0641\u064a\u0630 1935092757","translator_type":"none","protected":false,"verified":false,"followers_count":903,"friends_count":389,"listed_count":0,"favourites_count":2174,"statuses_count":33294,"created_at":"Thu Apr 16 12:12:15 +0000 2020","utc_offset":null,"time_zone":null,"geo_enabled":false,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"F5F8FA","profile_background_image_url":"","profile_background_image_url_https":"","profile_background_tile":false,"profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/1286520143431061504\/k2Jg3W0u_normal.jpg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/1286520143431061504\/k2Jg3W0u_normal.jpg","default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null,"withheld_in_countries":[]},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweeted_status":{"created_at":"Sat May 08 02:27:00 +0000 2021","id":1390855727116427270,"id_str":"1390855727116427270","text":"#\u0644\u0627_\u0644\u0644\u062a\u0637\u0639\u064a\u0645_\u0627\u0644\u0627\u062c\u0628\u0627\u0631\u064a\n\u0628\u0633\u0645 \u0627\u0644\u0644\u0647 \u062a\u0648\u0643\u0644\u062a \u0639\u0644\u0649 \u0627\u0644\u0644\u0647 \n\u0648\u0627\u0644\u0644\u0647 \u0643\u0631\u064a\u0645 \u064a\u0627\u0631\u0628 \u062a\u062a\u0642\u0641\u0644 \u0627\u0644\u064a\u0648\u0645 \u0628\u0648\u062c\u0648\u062f\u0627\u0644\u062e\u064a\u0631\u064a\u0646 \u0627\u0645\u062b\u0627\u0644\u0643\u0645\n \u064a\u0627\u0631\u0628 \u0633\u062e\u0631\u0644\u0647 \u0645\u0646 \u064a\u0642\u0641\u0644\u2026 https:\/\/t.co\/mpCKrtjMW4","display_text_range":[0,140],"source":"\u003ca href=\"http:\/\/twitter.com\/download\/android\" rel=\"nofollow\"\u003eTwitter for Android\u003c\/a\u003e","truncated":true,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":1338279194892115974,"id_str":"1338279194892115974","name":"\u0639\u0628\u062f\u0627\u0644\u0631\u062d\u0645\u0646 \u0645\u062d\u0645\u062f 2","screen_name":"aa86marat2","location":null,"url":null,"description":null,"translator_type":"none","protected":false,"verified":false,"followers_count":382,"friends_count":82,"listed_count":1,"favourites_count":43,"statuses_count":16494,"created_at":"Mon Dec 14 00:26:54 +0000 2020","utc_offset":null,"time_zone":null,"geo_enabled":false,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"F5F8FA","profile_background_image_url":"","profile_background_image_url_https":"","profile_background_tile":false,"profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/1356322484673519616\/7NSfbGye_normal.jpg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/1356322484673519616\/7NSfbGye_normal.jpg","default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null,"withheld_in_countries":[]},"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"extended_tweet":{"full_text":"#\u0644\u0627_\u0644\u0644\u062a\u0637\u0639\u064a\u0645_\u0627\u0644\u0627\u062c\u0628\u0627\u0631\u064a\n\u0628\u0633\u0645 \u0627\u0644\u0644\u0647 \u062a\u0648\u0643\u0644\u062a \u0639\u0644\u0649 \u0627\u0644\u0644\u0647 \n\u0648\u0627\u0644\u0644\u0647 \u0643\u0631\u064a\u0645 \u064a\u0627\u0631\u0628 \u062a\u062a\u0642\u0641\u0644 \u0627\u0644\u064a\u0648\u0645 \u0628\u0648\u062c\u0648\u062f\u0627\u0644\u062e\u064a\u0631\u064a\u0646 \u0627\u0645\u062b\u0627\u0644\u0643\u0645\n \u064a\u0627\u0631\u0628 \u0633\u062e\u0631\u0644\u0647 \u0645\u0646 \u064a\u0642\u0641\u0644\u0647\u0627\n\u0633\u062c\u064a\u0646 \u0639\u0645\u0631\u064756 \u0639\u0627\u0645\u0627 \u0645\u062a\u0632\u0648\u062c \u0644\u062f\u064a\u0647 \u0637\u0641\u0644 \u0645\u0633\u062c\u0648\u0646 \u0645\u0646\u0630\u0639\u0627\u0645\u064a\u0646 \u06484\u0623\u0634\u0647\u0631\n \u0645\u062a\u0628\u0642\u0649 \u0639\u0644\u064a\u0647 296200 \u0631\u064a\u0627\u0644\n\u0627\u0644\u0641\u0627\u062a\u0648\u0631\u0629\n1934638011\n\u0639\u0628\u0631 @\n\nhttps:\/\/t.co\/JjOHPoGLrJ\n\n\u0627\u0644\u062c\u0631\u0623\u0629_\u0641\u064a_\u0627\u0644\u0637\u0628\u0639\n\u0645\u0635\u0631\u0641_\u0627\u0644\u0631\u0627\u062c\u062d\u064a https:\/\/t.co\/mGuOo9qwZP","display_text_range":[0,276],"entities":{"hashtags":[{"text":"\u0644\u0627_\u0644\u0644\u062a\u0637\u0639\u064a\u0645_\u0627\u0644\u0627\u062c\u0628\u0627\u0631\u064a","indices":[0,20]}],"urls":[{"url":"https:\/\/t.co\/JjOHPoGLrJ","expanded_url":"https:\/\/Ehsan.sa\/referral\/29734F898E5CCB199DD92FD5CE8284C3B10081046BB13791AC72608BDD62B3D84020AC2B515E485C2E4B718A425C1A710438C7B814161E4197F6E7F4F73557AA","display_url":"Ehsan.sa\/referral\/29734\u2026","indices":[223,246]}],"user_mentions":[],"symbols":[],"media":[{"id":1390855708363698183,"id_str":"1390855708363698183","indices":[277,300],"media_url":"http:\/\/pbs.twimg.com\/media\/E01P5c7XsAcgFlQ.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E01P5c7XsAcgFlQ.jpg","url":"https:\/\/t.co\/mGuOo9qwZP","display_url":"pic.twitter.com\/mGuOo9qwZP","expanded_url":"https:\/\/twitter.com\/aa86marat2\/status\/1390855727116427270\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":554,"h":1200,"resize":"fit"},"large":{"w":720,"h":1560,"resize":"fit"},"small":{"w":314,"h":680,"resize":"fit"}}},{"id":1390855718069317649,"id_str":"1390855718069317649","indices":[277,300],"media_url":"http:\/\/pbs.twimg.com\/media\/E01P6BFXsBEhWDZ.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E01P6BFXsBEhWDZ.jpg","url":"https:\/\/t.co\/mGuOo9qwZP","display_url":"pic.twitter.com\/mGuOo9qwZP","expanded_url":"https:\/\/twitter.com\/aa86marat2\/status\/1390855727116427270\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"small":{"w":511,"h":680,"resize":"fit"},"large":{"w":901,"h":1200,"resize":"fit"},"medium":{"w":901,"h":1200,"resize":"fit"}}}]},"extended_entities":{"media":[{"id":1390855708363698183,"id_str":"1390855708363698183","indices":[277,300],"media_url":"http:\/\/pbs.twimg.com\/media\/E01P5c7XsAcgFlQ.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E01P5c7XsAcgFlQ.jpg","url":"https:\/\/t.co\/mGuOo9qwZP","display_url":"pic.twitter.com\/mGuOo9qwZP","expanded_url":"https:\/\/twitter.com\/aa86marat2\/status\/1390855727116427270\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":554,"h":1200,"resize":"fit"},"large":{"w":720,"h":1560,"resize":"fit"},"small":{"w":314,"h":680,"resize":"fit"}}},{"id":1390855718069317649,"id_str":"1390855718069317649","indices":[277,300],"media_url":"http:\/\/pbs.twimg.com\/media\/E01P6BFXsBEhWDZ.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E01P6BFXsBEhWDZ.jpg","url":"https:\/\/t.co\/mGuOo9qwZP","display_url":"pic.twitter.com\/mGuOo9qwZP","expanded_url":"https:\/\/twitter.com\/aa86marat2\/status\/1390855727116427270\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"small":{"w":511,"h":680,"resize":"fit"},"large":{"w":901,"h":1200,"resize":"fit"},"medium":{"w":901,"h":1200,"resize":"fit"}}}]}},"quote_count":0,"reply_count":1,"retweet_count":33,"favorite_count":2,"entities":{"hashtags":[{"text":"\u0644\u0627_\u0644\u0644\u062a\u0637\u0639\u064a\u0645_\u0627\u0644\u0627\u062c\u0628\u0627\u0631\u064a","indices":[0,20]}],"urls":[{"url":"https:\/\/t.co\/mpCKrtjMW4","expanded_url":"https:\/\/twitter.com\/i\/web\/status\/1390855727116427270","display_url":"twitter.com\/i\/web\/status\/1\u2026","indices":[117,140]}],"user_mentions":[],"symbols":[]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"low","lang":"ar"},"is_quote_status":false,"quote_count":0,"reply_count":0,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[{"text":"\u0644\u0627_\u0644\u0644\u062a\u0637\u0639\u064a\u0645_\u0627\u0644\u0627\u062c\u0628\u0627\u0631\u064a","indices":[16,36]}],"urls":[],"user_mentions":[{"screen_name":"aa86marat2","name":"\u0639\u0628\u062f\u0627\u0644\u0631\u062d\u0645\u0646 \u0645\u062d\u0645\u062f 2","id":1338279194892115974,"id_str":"1338279194892115974","indices":[3,14]}],"symbols":[]},"favorited":false,"retweeted":false,"filter_level":"low","lang":"ar","timestamp_ms":"1620466064719"}
{"created_at":"Sat May 08 09:27:44 +0000 2021","id":1390961607799296000,"id_str":"1390961607799296000","text":"RT @ariftgif: Tag ur bestie \/ fav person \/ @ third person https:\/\/t.co\/L4pGn5nAKJ","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":812488479607169024,"id_str":"812488479607169024","name":"\ud83c\udfae","screen_name":"bjeyyy_","location":null,"url":null,"description":"always 20 hihihi","translator_type":"none","protected":false,"verified":false,"followers_count":100,"friends_count":108,"listed_count":0,"favourites_count":2519,"statuses_count":48348,"created_at":"Sat Dec 24 02:42:03 +0000 2016","utc_offset":null,"time_zone":null,"geo_enabled":true,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"F5F8FA","profile_background_image_url":"","profile_background_image_url_https":"","profile_background_tile":false,"profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/1381271588394336256\/CnmfqTCN_normal.jpg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/1381271588394336256\/CnmfqTCN_normal.jpg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/812488479607169024\/1582382619","default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null,"withheld_in_countries":[]},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweeted_status":{"created_at":"Fri May 07 10:38:58 +0000 2021","id":1390617147198480388,"id_str":"1390617147198480388","text":"Tag ur bestie \/ fav person \/ @ third person https:\/\/t.co\/L4pGn5nAKJ","display_text_range":[0,43],"source":"\u003ca href=\"http:\/\/twitter.com\/download\/android\" rel=\"nofollow\"\u003eTwitter for Android\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":1188902968277385216,"id_str":"1188902968277385216","name":"arif | \u2606","screen_name":"ariftgif","location":"\ud835\udcb6\ud835\udcc3\ud835\udcbf\ud835\udcb6\ud835\udcc8\ud835\udcc2\ud835\udcb6\ud835\udcc7\ud835\udcb6","url":"https:\/\/vt.tiktok.com\/ZSEAw2WN\/","description":"ugly asf | a man with mullet hair","translator_type":"none","protected":false,"verified":false,"followers_count":2144,"friends_count":937,"listed_count":0,"favourites_count":20934,"statuses_count":19470,"created_at":"Mon Oct 28 19:39:09 +0000 2019","utc_offset":null,"time_zone":null,"geo_enabled":false,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"F5F8FA","profile_background_image_url":"","profile_background_image_url_https":"","profile_background_tile":false,"profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/1387751035041505282\/f_Q5rJ_B_normal.jpg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/1387751035041505282\/f_Q5rJ_B_normal.jpg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/1188902968277385216\/1619601431","default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null,"withheld_in_countries":[]},"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"quote_count":49,"reply_count":176,"retweet_count":1451,"favorite_count":3773,"entities":{"hashtags":[],"urls":[],"user_mentions":[],"symbols":[],"media":[{"id":1390617137362792450,"id_str":"1390617137362792450","indices":[44,67],"media_url":"http:\/\/pbs.twimg.com\/media\/E0x26ysUcAIYLJd.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E0x26ysUcAIYLJd.jpg","url":"https:\/\/t.co\/L4pGn5nAKJ","display_url":"pic.twitter.com\/L4pGn5nAKJ","expanded_url":"https:\/\/twitter.com\/ariftgif\/status\/1390617147198480388\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"small":{"w":680,"h":667,"resize":"fit"},"medium":{"w":1080,"h":1060,"resize":"fit"},"large":{"w":1080,"h":1060,"resize":"fit"}}}]},"extended_entities":{"media":[{"id":1390617137362792450,"id_str":"1390617137362792450","indices":[44,67],"media_url":"http:\/\/pbs.twimg.com\/media\/E0x26ysUcAIYLJd.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E0x26ysUcAIYLJd.jpg","url":"https:\/\/t.co\/L4pGn5nAKJ","display_url":"pic.twitter.com\/L4pGn5nAKJ","expanded_url":"https:\/\/twitter.com\/ariftgif\/status\/1390617147198480388\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"small":{"w":680,"h":667,"resize":"fit"},"medium":{"w":1080,"h":1060,"resize":"fit"},"large":{"w":1080,"h":1060,"resize":"fit"}}}]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"low","lang":"en"},"is_quote_status":false,"quote_count":0,"reply_count":0,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"urls":[],"user_mentions":[{"screen_name":"ariftgif","name":"arif | \u2606","id":1188902968277385216,"id_str":"1188902968277385216","indices":[3,12]}],"symbols":[],"media":[{"id":1390617137362792450,"id_str":"1390617137362792450","indices":[58,81],"media_url":"http:\/\/pbs.twimg.com\/media\/E0x26ysUcAIYLJd.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E0x26ysUcAIYLJd.jpg","url":"https:\/\/t.co\/L4pGn5nAKJ","display_url":"pic.twitter.com\/L4pGn5nAKJ","expanded_url":"https:\/\/twitter.com\/ariftgif\/status\/1390617147198480388\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"small":{"w":680,"h":667,"resize":"fit"},"medium":{"w":1080,"h":1060,"resize":"fit"},"large":{"w":1080,"h":1060,"resize":"fit"}},"source_status_id":1390617147198480388,"source_status_id_str":"1390617147198480388","source_user_id":1188902968277385216,"source_user_id_str":"1188902968277385216"}]},"extended_entities":{"media":[{"id":1390617137362792450,"id_str":"1390617137362792450","indices":[58,81],"media_url":"http:\/\/pbs.twimg.com\/media\/E0x26ysUcAIYLJd.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E0x26ysUcAIYLJd.jpg","url":"https:\/\/t.co\/L4pGn5nAKJ","display_url":"pic.twitter.com\/L4pGn5nAKJ","expanded_url":"https:\/\/twitter.com\/ariftgif\/status\/1390617147198480388\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"small":{"w":680,"h":667,"resize":"fit"},"medium":{"w":1080,"h":1060,"resize":"fit"},"large":{"w":1080,"h":1060,"resize":"fit"}},"source_status_id":1390617147198480388,"source_status_id_str":"1390617147198480388","source_user_id":1188902968277385216,"source_user_id_str":"1188902968277385216"}]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"low","lang":"en","timestamp_ms":"1620466064773"}
我是 spark 的新手，已经花了 3 天时间了解如何从嵌套 JSON Inside DStream 对象中获取价值。
您可以 map 将 json 字符串转换为您要提取的值的元组：
import json
def parse_json_string(str):
user = json.loads(str)["user"]
return (user["name"], user["followers_count"])
stream = socket_stream.window...
stream = stream.map(lambda str: parse_json_string(str))
stream.pprint()
ssc.start()
ssc.awaitTermination()
输出：
-------------------------------------------
Time: 2021-05-08 18:50:48
-------------------------------------------
('Nahoj Morts', 15)
('KISSMEMYFAN1106♡', 91307)
('om_mhamd99', 903)
('', 100)
我编写了以下代码来从 Tweepy API 流式传输数据。我正在获取流对象中的数据。但是获取不到streamp["user"]["followers_count"]但是不知道怎么获取。我也试过 jsonLines = lines.flatMap(lambda json_str:json.loads(json_str)) 但没有帮助。
from __future__ import print_function
from pyspark import SparkContext
from pyspark.streaming import StreamingContext
import json
sc = SparkContext()
ssc = StreamingContext(sc, 10) #Slide Interval of 10 sec
socket_stream = ssc.socketTextStream("localhost",4444)
stream = socket_stream.window(30) #window length 30 sec
stream.pprint()
ssc.start()
ssc.awaitTermination()
stream.pprint() 给了我以下 JSON.
{"created_at":"Sat May 08 09:27:43 +0000 2021","id":1390961604079067137,"id_str":"1390961604079067137","text":"The return of the king! This style has so many parameters that have to be correct and Stone nails em all! - Drinkin\u2026 https:\/\/t.co\/uCogsfW8NC","source":"\u003ca href=\"https:\/\/untappd.com\" rel=\"nofollow\"\u003eUntappd\u003c\/a\u003e","truncated":true,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":905439001,"id_str":"905439001","name":"Nahoj Morts","screen_name":"CraftBeerJunkie","location":null,"url":null,"description":null,"translator_type":"none","protected":false,"verified":false,"followers_count":15,"friends_count":113,"listed_count":0,"favourites_count":1,"statuses_count":1868,"created_at":"Fri Oct 26 06:36:01 +0000 2012","utc_offset":null,"time_zone":null,"geo_enabled":false,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/2765713047\/b92d58c569ad4739e67ef4d4e9a35780_normal.jpeg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/2765713047\/b92d58c569ad4739e67ef4d4e9a35780_normal.jpeg","default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null,"withheld_in_countries":[]},"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"extended_tweet":{"full_text":"The return of the king! This style has so many parameters that have to be correct and Stone nails em all! - Drinking a Stone Sublimely Self-Righteous Black IPA by @StoneBrewing @ Uggleberget \u2014 https:\/\/t.co\/a3XixD5teo","display_text_range":[0,217],"entities":{"hashtags":[],"urls":[{"url":"https:\/\/t.co\/a3XixD5teo","expanded_url":"https:\/\/untp.beer\/s\/c1025322433","display_url":"untp.beer\/s\/c1025322433","indices":[194,217]}],"user_mentions":[{"screen_name":"StoneBrewing","name":"Stone Brewing","id":16331259,"id_str":"16331259","indices":[163,176]}],"symbols":[]}},"quote_count":0,"reply_count":0,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"urls":[{"url":"https:\/\/t.co\/uCogsfW8NC","expanded_url":"https:\/\/twitter.com\/i\/web\/status\/1390961604079067137","display_url":"twitter.com\/i\/web\/status\/1\u2026","indices":[117,140]}],"user_mentions":[],"symbols":[]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"low","lang":"en","timestamp_ms":"1620466063886"}
{"created_at":"Sat May 08 09:27:43 +0000 2021","id":1390961604334919683,"id_str":"1390961604334919683","text":"[HQ] 210508 #KrisWu #Wuyifan @ GTSSC 2021 (Day 1) Cr.Fanbaobao(3) #PorscheRacerKrisWu \nhttps:\/\/t.co\/cEcAtzKNsM\u2026 https:\/\/t.co\/3B9uUS3mdu","display_text_range":[0,140],"source":"\u003ca href=\"https:\/\/mobile.twitter.com\" rel=\"nofollow\"\u003eTwitter Web App\u003c\/a\u003e","truncated":true,"in_reply_to_status_id":1390961304316387331,"in_reply_to_status_id_str":"1390961304316387331","in_reply_to_user_id":620222301,"in_reply_to_user_id_str":"620222301","in_reply_to_screen_name":"kissmemyfan_","user":{"id":620222301,"id_str":"620222301","name":"KISSMEMYFAN1106\u2661","screen_name":"kissmemyfan_","location":null,"url":"https:\/\/www.youtube.com\/channel\/UCtYrDyYwqZyQfYtCz79zPHQ\/featured","description":"for @KrisWu Kris Wu Yifan since 120627 \u2740notify in case of emergency,pls dm\u2740 fb: https:\/\/www.facebook.com\/groups\/1051879991503131\/","translator_type":"regular","protected":false,"verified":false,"followers_count":91307,"friends_count":24,"listed_count":433,"favourites_count":14194,"statuses_count":95476,"created_at":"Wed Jun 27 16:34:02 +0000 2012","utc_offset":null,"time_zone":null,"geo_enabled":true,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"FFCC4D","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":true,"profile_link_color":"E81C4F","profile_sidebar_border_color":"FFFFFF","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":false,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/1252957871802707974\/RjNGNnly_normal.jpg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/1252957871802707974\/RjNGNnly_normal.jpg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/620222301\/1587563426","default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null,"withheld_in_countries":[]},"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"extended_tweet":{"full_text":"[HQ] 210508 #KrisWu #Wuyifan @ GTSSC 2021 (Day 1) Cr.Fanbaobao(3) #PorscheRacerKrisWu \nhttps:\/\/t.co\/cEcAtzKNsM\nhttps:\/\/t.co\/PNbPqE88Y3 https:\/\/t.co\/8WPhfpHiyO","display_text_range":[0,134],"entities":{"hashtags":[{"text":"KrisWu","indices":[12,19]},{"text":"Wuyifan","indices":[20,28]},{"text":"PorscheRacerKrisWu","indices":[66,85]}],"urls":[{"url":"https:\/\/t.co\/cEcAtzKNsM","expanded_url":"https:\/\/wx1.sinaimg.cn\/large\/00688bwvly1gqb3c1cdbuj31l82bpe81.jpg","display_url":"wx1.sinaimg.cn\/large\/00688bwv\u2026","indices":[87,110]},{"url":"https:\/\/t.co\/PNbPqE88Y3","expanded_url":"https:\/\/wx4.sinaimg.cn\/large\/00688bwvly1gqb3c1mc9hj31ay1wb7ji.jpg","display_url":"wx4.sinaimg.cn\/large\/00688bwv\u2026","indices":[111,134]}],"user_mentions":[],"symbols":[],"media":[{"id":1390961423400857602,"id_str":"1390961423400857602","indices":[135,158],"media_url":"http:\/\/pbs.twimg.com\/media\/E02wC4IUcAI4F-O.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E02wC4IUcAI4F-O.jpg","url":"https:\/\/t.co\/8WPhfpHiyO","display_url":"pic.twitter.com\/8WPhfpHiyO","expanded_url":"https:\/\/twitter.com\/kissmemyfan_\/status\/1390961604334919683\/photo\/1","type":"photo","sizes":{"small":{"w":465,"h":680,"resize":"fit"},"thumb":{"w":150,"h":150,"resize":"crop"},"large":{"w":1024,"h":1498,"resize":"fit"},"medium":{"w":820,"h":1200,"resize":"fit"}}},{"id":1390961423417704450,"id_str":"1390961423417704450","indices":[135,158],"media_url":"http:\/\/pbs.twimg.com\/media\/E02wC4MVgAI8ZHZ.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E02wC4MVgAI8ZHZ.jpg","url":"https:\/\/t.co\/8WPhfpHiyO","display_url":"pic.twitter.com\/8WPhfpHiyO","expanded_url":"https:\/\/twitter.com\/kissmemyfan_\/status\/1390961604334919683\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":825,"h":1200,"resize":"fit"},"small":{"w":467,"h":680,"resize":"fit"},"large":{"w":1024,"h":1490,"resize":"fit"}}}]},"extended_entities":{"media":[{"id":1390961423400857602,"id_str":"1390961423400857602","indices":[135,158],"media_url":"http:\/\/pbs.twimg.com\/media\/E02wC4IUcAI4F-O.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E02wC4IUcAI4F-O.jpg","url":"https:\/\/t.co\/8WPhfpHiyO","display_url":"pic.twitter.com\/8WPhfpHiyO","expanded_url":"https:\/\/twitter.com\/kissmemyfan_\/status\/1390961604334919683\/photo\/1","type":"photo","sizes":{"small":{"w":465,"h":680,"resize":"fit"},"thumb":{"w":150,"h":150,"resize":"crop"},"large":{"w":1024,"h":1498,"resize":"fit"},"medium":{"w":820,"h":1200,"resize":"fit"}}},{"id":1390961423417704450,"id_str":"1390961423417704450","indices":[135,158],"media_url":"http:\/\/pbs.twimg.com\/media\/E02wC4MVgAI8ZHZ.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E02wC4MVgAI8ZHZ.jpg","url":"https:\/\/t.co\/8WPhfpHiyO","display_url":"pic.twitter.com\/8WPhfpHiyO","expanded_url":"https:\/\/twitter.com\/kissmemyfan_\/status\/1390961604334919683\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":825,"h":1200,"resize":"fit"},"small":{"w":467,"h":680,"resize":"fit"},"large":{"w":1024,"h":1490,"resize":"fit"}}}]}},"quote_count":0,"reply_count":0,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[{"text":"KrisWu","indices":[12,19]},{"text":"Wuyifan","indices":[20,28]},{"text":"PorscheRacerKrisWu","indices":[66,85]}],"urls":[{"url":"https:\/\/t.co\/cEcAtzKNsM","expanded_url":"https:\/\/wx1.sinaimg.cn\/large\/00688bwvly1gqb3c1cdbuj31l82bpe81.jpg","display_url":"wx1.sinaimg.cn\/large\/00688bwv\u2026","indices":[87,110]},{"url":"https:\/\/t.co\/3B9uUS3mdu","expanded_url":"https:\/\/twitter.com\/i\/web\/status\/1390961604334919683","display_url":"twitter.com\/i\/web\/status\/1\u2026","indices":[112,135]}],"user_mentions":[],"symbols":[]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"low","lang":"ht","timestamp_ms":"1620466063947"}
{"created_at":"Sat May 08 09:27:44 +0000 2021","id":1390961607572865031,"id_str":"1390961607572865031","text":"RT @aa86marat2: #\u0644\u0627_\u0644\u0644\u062a\u0637\u0639\u064a\u0645_\u0627\u0644\u0627\u062c\u0628\u0627\u0631\u064a\n\u0628\u0633\u0645 \u0627\u0644\u0644\u0647 \u062a\u0648\u0643\u0644\u062a \u0639\u0644\u0649 \u0627\u0644\u0644\u0647 \n\u0648\u0627\u0644\u0644\u0647 \u0643\u0631\u064a\u0645 \u064a\u0627\u0631\u0628 \u062a\u062a\u0642\u0641\u0644 \u0627\u0644\u064a\u0648\u0645 \u0628\u0648\u062c\u0648\u062f\u0627\u0644\u062e\u064a\u0631\u064a\u0646 \u0627\u0645\u062b\u0627\u0644\u0643\u0645\n \u064a\u0627\u0631\u0628 \u0633\u062e\u0631\u0644\u0647 \u0645\u0646 \u064a\u0642\u0641\u0644\u0647\u0627\n\u0633\u062c\u064a\u0646\u2026","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":1250758860660527105,"id_str":"1250758860660527105","name":"om_mhamd99","screen_name":"Mhamd99Om","location":null,"url":null,"description":"\u064a\u0627\u0631\u0628 \u2728\ud83e\udd32\ud83c\udffb\u0628\u0627\u0631\u0643 \u0644\u064a \u0641\u064a\u0645\u0627 \u0623\u0639\u0637\u062a\u0646\u064a \u0648\u0641\u0631\u062c \u0647\u0645\u064a \u0628\u0642\u0636\u0627\u0621 \u062f\u064a\u0646 \u0648\u0627\u0639\u062a\u064a \u0639\u0644\u064a \u0630\u0644\u0643 \u0641\u0627\u062a\u0648\u0631\u0647 \u062a\u0646\u0641\u064a\u0630 1935092757","translator_type":"none","protected":false,"verified":false,"followers_count":903,"friends_count":389,"listed_count":0,"favourites_count":2174,"statuses_count":33294,"created_at":"Thu Apr 16 12:12:15 +0000 2020","utc_offset":null,"time_zone":null,"geo_enabled":false,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"F5F8FA","profile_background_image_url":"","profile_background_image_url_https":"","profile_background_tile":false,"profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/1286520143431061504\/k2Jg3W0u_normal.jpg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/1286520143431061504\/k2Jg3W0u_normal.jpg","default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null,"withheld_in_countries":[]},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweeted_status":{"created_at":"Sat May 08 02:27:00 +0000 2021","id":1390855727116427270,"id_str":"1390855727116427270","text":"#\u0644\u0627_\u0644\u0644\u062a\u0637\u0639\u064a\u0645_\u0627\u0644\u0627\u062c\u0628\u0627\u0631\u064a\n\u0628\u0633\u0645 \u0627\u0644\u0644\u0647 \u062a\u0648\u0643\u0644\u062a \u0639\u0644\u0649 \u0627\u0644\u0644\u0647 \n\u0648\u0627\u0644\u0644\u0647 \u0643\u0631\u064a\u0645 \u064a\u0627\u0631\u0628 \u062a\u062a\u0642\u0641\u0644 \u0627\u0644\u064a\u0648\u0645 \u0628\u0648\u062c\u0648\u062f\u0627\u0644\u062e\u064a\u0631\u064a\u0646 \u0627\u0645\u062b\u0627\u0644\u0643\u0645\n \u064a\u0627\u0631\u0628 \u0633\u062e\u0631\u0644\u0647 \u0645\u0646 \u064a\u0642\u0641\u0644\u2026 https:\/\/t.co\/mpCKrtjMW4","display_text_range":[0,140],"source":"\u003ca href=\"http:\/\/twitter.com\/download\/android\" rel=\"nofollow\"\u003eTwitter for Android\u003c\/a\u003e","truncated":true,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":1338279194892115974,"id_str":"1338279194892115974","name":"\u0639\u0628\u062f\u0627\u0644\u0631\u062d\u0645\u0646 \u0645\u062d\u0645\u062f 2","screen_name":"aa86marat2","location":null,"url":null,"description":null,"translator_type":"none","protected":false,"verified":false,"followers_count":382,"friends_count":82,"listed_count":1,"favourites_count":43,"statuses_count":16494,"created_at":"Mon Dec 14 00:26:54 +0000 2020","utc_offset":null,"time_zone":null,"geo_enabled":false,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"F5F8FA","profile_background_image_url":"","profile_background_image_url_https":"","profile_background_tile":false,"profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/1356322484673519616\/7NSfbGye_normal.jpg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/1356322484673519616\/7NSfbGye_normal.jpg","default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null,"withheld_in_countries":[]},"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"extended_tweet":{"full_text":"#\u0644\u0627_\u0644\u0644\u062a\u0637\u0639\u064a\u0645_\u0627\u0644\u0627\u062c\u0628\u0627\u0631\u064a\n\u0628\u0633\u0645 \u0627\u0644\u0644\u0647 \u062a\u0648\u0643\u0644\u062a \u0639\u0644\u0649 \u0627\u0644\u0644\u0647 \n\u0648\u0627\u0644\u0644\u0647 \u0643\u0631\u064a\u0645 \u064a\u0627\u0631\u0628 \u062a\u062a\u0642\u0641\u0644 \u0627\u0644\u064a\u0648\u0645 \u0628\u0648\u062c\u0648\u062f\u0627\u0644\u062e\u064a\u0631\u064a\u0646 \u0627\u0645\u062b\u0627\u0644\u0643\u0645\n \u064a\u0627\u0631\u0628 \u0633\u062e\u0631\u0644\u0647 \u0645\u0646 \u064a\u0642\u0641\u0644\u0647\u0627\n\u0633\u062c\u064a\u0646 \u0639\u0645\u0631\u064756 \u0639\u0627\u0645\u0627 \u0645\u062a\u0632\u0648\u062c \u0644\u062f\u064a\u0647 \u0637\u0641\u0644 \u0645\u0633\u062c\u0648\u0646 \u0645\u0646\u0630\u0639\u0627\u0645\u064a\u0646 \u06484\u0623\u0634\u0647\u0631\n \u0645\u062a\u0628\u0642\u0649 \u0639\u0644\u064a\u0647 296200 \u0631\u064a\u0627\u0644\n\u0627\u0644\u0641\u0627\u062a\u0648\u0631\u0629\n1934638011\n\u0639\u0628\u0631 @\n\nhttps:\/\/t.co\/JjOHPoGLrJ\n\n\u0627\u0644\u062c\u0631\u0623\u0629_\u0641\u064a_\u0627\u0644\u0637\u0628\u0639\n\u0645\u0635\u0631\u0641_\u0627\u0644\u0631\u0627\u062c\u062d\u064a https:\/\/t.co\/mGuOo9qwZP","display_text_range":[0,276],"entities":{"hashtags":[{"text":"\u0644\u0627_\u0644\u0644\u062a\u0637\u0639\u064a\u0645_\u0627\u0644\u0627\u062c\u0628\u0627\u0631\u064a","indices":[0,20]}],"urls":[{"url":"https:\/\/t.co\/JjOHPoGLrJ","expanded_url":"https:\/\/Ehsan.sa\/referral\/29734F898E5CCB199DD92FD5CE8284C3B10081046BB13791AC72608BDD62B3D84020AC2B515E485C2E4B718A425C1A710438C7B814161E4197F6E7F4F73557AA","display_url":"Ehsan.sa\/referral\/29734\u2026","indices":[223,246]}],"user_mentions":[],"symbols":[],"media":[{"id":1390855708363698183,"id_str":"1390855708363698183","indices":[277,300],"media_url":"http:\/\/pbs.twimg.com\/media\/E01P5c7XsAcgFlQ.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E01P5c7XsAcgFlQ.jpg","url":"https:\/\/t.co\/mGuOo9qwZP","display_url":"pic.twitter.com\/mGuOo9qwZP","expanded_url":"https:\/\/twitter.com\/aa86marat2\/status\/1390855727116427270\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":554,"h":1200,"resize":"fit"},"large":{"w":720,"h":1560,"resize":"fit"},"small":{"w":314,"h":680,"resize":"fit"}}},{"id":1390855718069317649,"id_str":"1390855718069317649","indices":[277,300],"media_url":"http:\/\/pbs.twimg.com\/media\/E01P6BFXsBEhWDZ.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E01P6BFXsBEhWDZ.jpg","url":"https:\/\/t.co\/mGuOo9qwZP","display_url":"pic.twitter.com\/mGuOo9qwZP","expanded_url":"https:\/\/twitter.com\/aa86marat2\/status\/1390855727116427270\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"small":{"w":511,"h":680,"resize":"fit"},"large":{"w":901,"h":1200,"resize":"fit"},"medium":{"w":901,"h":1200,"resize":"fit"}}}]},"extended_entities":{"media":[{"id":1390855708363698183,"id_str":"1390855708363698183","indices":[277,300],"media_url":"http:\/\/pbs.twimg.com\/media\/E01P5c7XsAcgFlQ.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E01P5c7XsAcgFlQ.jpg","url":"https:\/\/t.co\/mGuOo9qwZP","display_url":"pic.twitter.com\/mGuOo9qwZP","expanded_url":"https:\/\/twitter.com\/aa86marat2\/status\/1390855727116427270\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":554,"h":1200,"resize":"fit"},"large":{"w":720,"h":1560,"resize":"fit"},"small":{"w":314,"h":680,"resize":"fit"}}},{"id":1390855718069317649,"id_str":"1390855718069317649","indices":[277,300],"media_url":"http:\/\/pbs.twimg.com\/media\/E01P6BFXsBEhWDZ.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E01P6BFXsBEhWDZ.jpg","url":"https:\/\/t.co\/mGuOo9qwZP","display_url":"pic.twitter.com\/mGuOo9qwZP","expanded_url":"https:\/\/twitter.com\/aa86marat2\/status\/1390855727116427270\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"small":{"w":511,"h":680,"resize":"fit"},"large":{"w":901,"h":1200,"resize":"fit"},"medium":{"w":901,"h":1200,"resize":"fit"}}}]}},"quote_count":0,"reply_count":1,"retweet_count":33,"favorite_count":2,"entities":{"hashtags":[{"text":"\u0644\u0627_\u0644\u0644\u062a\u0637\u0639\u064a\u0645_\u0627\u0644\u0627\u062c\u0628\u0627\u0631\u064a","indices":[0,20]}],"urls":[{"url":"https:\/\/t.co\/mpCKrtjMW4","expanded_url":"https:\/\/twitter.com\/i\/web\/status\/1390855727116427270","display_url":"twitter.com\/i\/web\/status\/1\u2026","indices":[117,140]}],"user_mentions":[],"symbols":[]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"low","lang":"ar"},"is_quote_status":false,"quote_count":0,"reply_count":0,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[{"text":"\u0644\u0627_\u0644\u0644\u062a\u0637\u0639\u064a\u0645_\u0627\u0644\u0627\u062c\u0628\u0627\u0631\u064a","indices":[16,36]}],"urls":[],"user_mentions":[{"screen_name":"aa86marat2","name":"\u0639\u0628\u062f\u0627\u0644\u0631\u062d\u0645\u0646 \u0645\u062d\u0645\u062f 2","id":1338279194892115974,"id_str":"1338279194892115974","indices":[3,14]}],"symbols":[]},"favorited":false,"retweeted":false,"filter_level":"low","lang":"ar","timestamp_ms":"1620466064719"}
{"created_at":"Sat May 08 09:27:44 +0000 2021","id":1390961607799296000,"id_str":"1390961607799296000","text":"RT @ariftgif: Tag ur bestie \/ fav person \/ @ third person https:\/\/t.co\/L4pGn5nAKJ","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":812488479607169024,"id_str":"812488479607169024","name":"\ud83c\udfae","screen_name":"bjeyyy_","location":null,"url":null,"description":"always 20 hihihi","translator_type":"none","protected":false,"verified":false,"followers_count":100,"friends_count":108,"listed_count":0,"favourites_count":2519,"statuses_count":48348,"created_at":"Sat Dec 24 02:42:03 +0000 2016","utc_offset":null,"time_zone":null,"geo_enabled":true,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"F5F8FA","profile_background_image_url":"","profile_background_image_url_https":"","profile_background_tile":false,"profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/1381271588394336256\/CnmfqTCN_normal.jpg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/1381271588394336256\/CnmfqTCN_normal.jpg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/812488479607169024\/1582382619","default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null,"withheld_in_countries":[]},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweeted_status":{"created_at":"Fri May 07 10:38:58 +0000 2021","id":1390617147198480388,"id_str":"1390617147198480388","text":"Tag ur bestie \/ fav person \/ @ third person https:\/\/t.co\/L4pGn5nAKJ","display_text_range":[0,43],"source":"\u003ca href=\"http:\/\/twitter.com\/download\/android\" rel=\"nofollow\"\u003eTwitter for Android\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":1188902968277385216,"id_str":"1188902968277385216","name":"arif | \u2606","screen_name":"ariftgif","location":"\ud835\udcb6\ud835\udcc3\ud835\udcbf\ud835\udcb6\ud835\udcc8\ud835\udcc2\ud835\udcb6\ud835\udcc7\ud835\udcb6","url":"https:\/\/vt.tiktok.com\/ZSEAw2WN\/","description":"ugly asf | a man with mullet hair","translator_type":"none","protected":false,"verified":false,"followers_count":2144,"friends_count":937,"listed_count":0,"favourites_count":20934,"statuses_count":19470,"created_at":"Mon Oct 28 19:39:09 +0000 2019","utc_offset":null,"time_zone":null,"geo_enabled":false,"lang":null,"contributors_enabled":false,"is_translator":false,"profile_background_color":"F5F8FA","profile_background_image_url":"","profile_background_image_url_https":"","profile_background_tile":false,"profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/1387751035041505282\/f_Q5rJ_B_normal.jpg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/1387751035041505282\/f_Q5rJ_B_normal.jpg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/1188902968277385216\/1619601431","default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null,"withheld_in_countries":[]},"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"quote_count":49,"reply_count":176,"retweet_count":1451,"favorite_count":3773,"entities":{"hashtags":[],"urls":[],"user_mentions":[],"symbols":[],"media":[{"id":1390617137362792450,"id_str":"1390617137362792450","indices":[44,67],"media_url":"http:\/\/pbs.twimg.com\/media\/E0x26ysUcAIYLJd.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E0x26ysUcAIYLJd.jpg","url":"https:\/\/t.co\/L4pGn5nAKJ","display_url":"pic.twitter.com\/L4pGn5nAKJ","expanded_url":"https:\/\/twitter.com\/ariftgif\/status\/1390617147198480388\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"small":{"w":680,"h":667,"resize":"fit"},"medium":{"w":1080,"h":1060,"resize":"fit"},"large":{"w":1080,"h":1060,"resize":"fit"}}}]},"extended_entities":{"media":[{"id":1390617137362792450,"id_str":"1390617137362792450","indices":[44,67],"media_url":"http:\/\/pbs.twimg.com\/media\/E0x26ysUcAIYLJd.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E0x26ysUcAIYLJd.jpg","url":"https:\/\/t.co\/L4pGn5nAKJ","display_url":"pic.twitter.com\/L4pGn5nAKJ","expanded_url":"https:\/\/twitter.com\/ariftgif\/status\/1390617147198480388\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"small":{"w":680,"h":667,"resize":"fit"},"medium":{"w":1080,"h":1060,"resize":"fit"},"large":{"w":1080,"h":1060,"resize":"fit"}}}]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"low","lang":"en"},"is_quote_status":false,"quote_count":0,"reply_count":0,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"urls":[],"user_mentions":[{"screen_name":"ariftgif","name":"arif | \u2606","id":1188902968277385216,"id_str":"1188902968277385216","indices":[3,12]}],"symbols":[],"media":[{"id":1390617137362792450,"id_str":"1390617137362792450","indices":[58,81],"media_url":"http:\/\/pbs.twimg.com\/media\/E0x26ysUcAIYLJd.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E0x26ysUcAIYLJd.jpg","url":"https:\/\/t.co\/L4pGn5nAKJ","display_url":"pic.twitter.com\/L4pGn5nAKJ","expanded_url":"https:\/\/twitter.com\/ariftgif\/status\/1390617147198480388\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"small":{"w":680,"h":667,"resize":"fit"},"medium":{"w":1080,"h":1060,"resize":"fit"},"large":{"w":1080,"h":1060,"resize":"fit"}},"source_status_id":1390617147198480388,"source_status_id_str":"1390617147198480388","source_user_id":1188902968277385216,"source_user_id_str":"1188902968277385216"}]},"extended_entities":{"media":[{"id":1390617137362792450,"id_str":"1390617137362792450","indices":[58,81],"media_url":"http:\/\/pbs.twimg.com\/media\/E0x26ysUcAIYLJd.jpg","media_url_https":"https:\/\/pbs.twimg.com\/media\/E0x26ysUcAIYLJd.jpg","url":"https:\/\/t.co\/L4pGn5nAKJ","display_url":"pic.twitter.com\/L4pGn5nAKJ","expanded_url":"https:\/\/twitter.com\/ariftgif\/status\/1390617147198480388\/photo\/1","type":"photo","sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"small":{"w":680,"h":667,"resize":"fit"},"medium":{"w":1080,"h":1060,"resize":"fit"},"large":{"w":1080,"h":1060,"resize":"fit"}},"source_status_id":1390617147198480388,"source_status_id_str":"1390617147198480388","source_user_id":1188902968277385216,"source_user_id_str":"1188902968277385216"}]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"low","lang":"en","timestamp_ms":"1620466064773"}
我是 spark 的新手，已经花了 3 天时间了解如何从嵌套 JSON Inside DStream 对象中获取价值。
您可以 map 将 json 字符串转换为您要提取的值的元组：
import json
def parse_json_string(str):
user = json.loads(str)["user"]
return (user["name"], user["followers_count"])
stream = socket_stream.window...
stream = stream.map(lambda str: parse_json_string(str))
stream.pprint()
ssc.start()
ssc.awaitTermination()
输出：
-------------------------------------------
Time: 2021-05-08 18:50:48
-------------------------------------------
('Nahoj Morts', 15)
('KISSMEMYFAN1106♡', 91307)
('om_mhamd99', 903)
('', 100)