tags: Python 基礎
1.Request Data:
透過ajax發送比較複雜的請求，加入Request data(也可以稱為Request Payload/Request Body)的觀念
2.觀察medium.com是透過哪個ajax去取得資料
記得要用無痕開Chrome
XHR 下面左邊有很多動作一個一個點，可以用Preview看看回傳什麼，找到title之後，針對這一個物件做處理
3.Headers
在Headers底下可以找到 Request URL，以及 Request Payload
Request Payload底下有一個 Show More，可以點開來看到全部的資料，一定要點開，等等複製的時候是要複製完整的 Request Payload
範例程式碼如下:
import urllib.request as req
import json
url = "https://medium.com/_/graphql"
# Create Request Payload
request_payload = {"operationName": "ExtendedFeedQuery", "variables": {"items": [{"postId": "54df4cd269f", "topicId": ""}, {"postId": "ba95322fe4d5", "topicId": ""}, {"postId": "ced9c91fe1d1", "topicId": ""}, {"postId": "9fadbd448b33", "topicId": ""}, {"postId": "f9e98cfe0ac", "topicId": ""}, {"postId": "36d48870776b", "topicId": ""}, {"postId": "a49ef409e400", "topicId": ""}, {"postId": "7c21e9245cff", "topicId": ""}, {"postId": "d20f8a7136bf", "topicId": ""}, {"postId": "4f4943dfcb03", "topicId": ""}, {"postId": "238f22c26796", "topicId": ""}, {"postId": "ec3023a30af2", "topicId": ""}, {"postId": "d9d7eef37e89", "topicId": ""}, {"postId": "462a1869045f", "topicId": ""}, {"postId": "81b70227c748", "topicId": ""}, {"postId": "f8a95f6d17f", "topicId": ""}, {"postId": "6bf6c5a5ce79", "topicId": ""}, {"postId": "a08336a012bc", "topicId": ""}, {"postId": "2338d5e94c3d", "topicId": ""}, {
"postId": "af302607060", "topicId": ""}]}, "query": "query ExtendedFeedQuery($items: [ExtendedFeedItemOptions!]!) {\n extendedFeedItems(items: $items) {\n post {\n ...PostListModulePostPreviewData\n __typename\n }\n metadata {\n topic {\n id\n name\n __typename\n }\n __typename\n }\n __typename\n }\n}\n\nfragment PostListModulePostPreviewData on Post {\n id\n firstPublishedAt\n readingTime\n createdAt\n mediumUrl\n previewImage {\n id\n __typename\n }\n title\n collection {\n id\n domain\n slug\n name\n navItems {\n url\n __typename\n }\n logo {\n id\n __typename\n }\n avatar {\n id\n __typename\n }\n __typename\n }\n creator {\n id\n name\n username\n imageId\n mediumMemberAt\n ...userUrl_user\n __typename\n }\n visibility\n isProxyPost\n isLocked\n ...HomeFeedItem_post\n ...HomeTrendingModule_post\n __typename\n}\n\nfragment HomeFeedItem_post on Post {\n __typename\n id\n title\n firstPublishedAt\n mediumUrl\n collection {\n id\n name\n domain\n logo {\n id\n __typename\n }\n __typename\n }\n creator {\n id\n name\n username\n imageId\n mediumMemberAt\n __typename\n }\n previewImage {\n id\n __typename\n }\n previewContent {\n subtitle\n __typename\n }\n readingTime\n tags {\n ...TopicPill_tag\n __typename\n }\n ...BookmarkButton_post\n ...CreatorActionOverflowPopover_post\n ...PostPresentationTracker_post\n ...PostPreviewAvatar_post\n}\n\nfragment TopicPill_tag on Tag {\n __typename\n id\n displayTitle\n}\n\nfragment BookmarkButton_post on Post {\n visibility\n ...SusiClickable_post\n ...AddToCatalogBookmarkButton_post\n __typename\n id\n}\n\nfragment SusiClickable_post on Post {\n id\n mediumUrl\n ...SusiContainer_post\n __typename\n}\n\nfragment SusiContainer_post on Post {\n id\n __typename\n}\n\nfragment AddToCatalogBookmarkButton_post on Post {\n ...AddToCatalogBase_post\n __typename\n id\n}\n\nfragment AddToCatalogBase_post on Post {\n id\n viewerEdge {\n catalogsConnection {\n catalogsContainingThis(type: LISTS) {\n catalogId\n catalogItemIds\n __typename\n }\n predefinedContainingThis {\n catalogId\n predefined\n catalogItemIds\n __typename\n }\n __typename\n }\n ...editCatalogItemsMutation_postViewerEdge\n ...useAddItemToPredefinedCatalog_postViewerEdge\n __typename\n id\n }\n ...WithToggleInsideCatalog_post\n __typename\n}\n\nfragment useAddItemToPredefinedCatalog_postViewerEdge on PostViewerEdge {\n id\n catalogsConnection {\n predefinedContainingThis {\n catalogId\n version\n predefined\n catalogItemIds\n __typename\n }\n __typename\n }\n __typename\n}\n\nfragment editCatalogItemsMutation_postViewerEdge on PostViewerEdge {\n id\n catalogsConnection {\n catalogsContainingThis(type: LISTS) {\n catalogId\n version\n catalogItemIds\n __typename\n }\n predefinedContainingThis {\n catalogId\n predefined\n version\n catalogItemIds\n __typename\n }\n __typename\n }\n __typename\n}\n\nfragment WithToggleInsideCatalog_post on Post {\n id\n viewerEdge {\n catalogsConnection {\n catalogsContainingThis(type: LISTS) {\n catalogId\n __typename\n }\n predefinedContainingThis {\n predefined\n __typename\n }\n __typename\n }\n __typename\n id\n }\n __typename\n}\n\nfragment CreatorActionOverflowPopover_post on Post {\n allowResponses\n id\n statusForCollection\n isLocked\n isPublished\n clapCount\n mediumUrl\n pinnedAt\n pinnedByCreatorAt\n curationEligibleAt\n mediumUrl\n responseDistribution\n visibility\n ...useIsPinnedInContext_post\n pendingCollection {\n id\n name\n creator {\n id\n __typename\n }\n avatar {\n id\n __typename\n }\n domain\n slug\n __typename\n }\n creator {\n id\n ...MutePopoverOptions_creator\n ...auroraHooks_publisher\n __typename\n }\n collection {\n id\n name\n creator {\n id\n __typename\n }\n avatar {\n id\n __typename\n }\n domain\n slug\n ...MutePopoverOptions_collection\n ...auroraHooks_publisher\n __typename\n }\n ...ClapMutation_post\n __typename\n}\n\nfragment MutePopoverOptions_creator on User {\n id\n __typename\n}\n\nfragment MutePopoverOptions_collection on Collection {\n id\n __typename\n}\n\nfragment ClapMutation_post on Post {\n __typename\n id\n clapCount\n ...MultiVoteCount_post\n}\n\nfragment MultiVoteCount_post on Post {\n id\n ...PostVotersNetwork_post\n __typename\n}\n\nfragment PostVotersNetwork_post on Post {\n id\n voterCount\n recommenders {\n name\n __typename\n }\n __typename\n}\n\nfragment useIsPinnedInContext_post on Post {\n id\n collection {\n id\n __typename\n }\n pendingCollection {\n id\n __typename\n }\n pinnedAt\n pinnedByCreatorAt\n __typename\n}\n\nfragment auroraHooks_publisher on Publisher {\n __typename\n ... on Collection {\n isAuroraEligible\n isAuroraVisible\n viewerEdge {\n id\n isEditor\n __typename\n }\n __typename\n id\n }\n ... on User {\n isAuroraVisible\n __typename\n id\n }\n}\n\nfragment PostPresentationTracker_post on Post {\n id\n visibility\n previewContent {\n isFullContent\n __typename\n }\n collection {\n id\n slug\n __typename\n }\n __typename\n}\n\nfragment PostPreviewAvatar_post on Post {\n __typename\n id\n collection {\n id\n name\n ...CollectionAvatar_collection\n __typename\n }\n creator {\n id\n username\n name\n ...UserAvatar_user\n ...userUrl_user\n __typename\n }\n}\n\nfragment CollectionAvatar_collection on Collection {\n name\n avatar {\n id\n __typename\n }\n ...collectionUrl_collection\n __typename\n id\n}\n\nfragment collectionUrl_collection on Collection {\n id\n domain\n slug\n __typename\n}\n\nfragment UserAvatar_user on User {\n __typename\n id\n imageId\n mediumMemberAt\n name\n username\n ...userUrl_user\n}\n\nfragment userUrl_user on User {\n __typename\n id\n customDomainState {\n live {\n domain\n __typename\n }\n __typename\n }\n hasSubdomain\n username\n}\n\nfragment HomeTrendingModule_post on Post {\n id\n ...HomeTrendingPostPreview_post\n __typename\n}\n\nfragment HomeTrendingPostPreview_post on Post {\n id\n title\n mediumUrl\n readingTime\n firstPublishedAt\n ...PostPreviewAvatar_post\n ...PostPresentationTracker_post\n __typename\n}\n"}
# Convert Request Payload to Json string
str_request_payload = json.dumps(request_payload).encode("utf-8")
# Create Request Object
# content-type => 告知後端API我要傳入的request_payload是什麼型態
# req.Request 第三個參數data 放Request Payload
requestItem = req.Request(url, headers={
"content-type": "application/json",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36"
}, data=str_request_payload)
with req.urlopen(requestItem) as response:
# result type is <class 'str'>
result = response.read().decode("utf-8")
# print(type(result))
# convert str to dic
result_dic = json.loads(result)
# print(result_dic["data"]["extendedFeedItems"][0]["post"]["title"])
# print all title
for item in result_dic["data"]["extendedFeedItems"]:
print(item["post"]["title"])
![DAY40：Are they the "same"?](https://birdbirdmurmur.coderbridge.io/2023/08/22/javascript/?utm_source=coderbridge-com&utm_medium=indie_related_post_img&utm_campaign=22 Python - Web Crawler Request Data_DAY40：Are they the "same"?_@birdbirdmurmur_https://static.coderbridge.com/images/covers/default-post-cover-1.jpg)
