]*class="number"[^>]*>(\d+)', row)
bid = re.search(r'', row)
title = re.search(r"itemprop='name'[^>]*>([^<]+)", row)
author = re.search(r'class="authorName"[^>]*>]*>([^<]+)', row)
avg = re.search(r'(\d+\.\d+)\s*avg rating', row)
cnt = re.search(r'(\d[\d,]*)\s*rating', row)
if not (bid and title):
continue
results.append({
'rank': int(rank.group(1)) if rank else None,
'book_id': bid.group(1),
'title': title.group(1).strip(),
'author': author.group(1).strip() if author else None,
'avg_rating': float(avg.group(1)) if avg else None,
'ratings_count':cnt.group(1).replace(',', '') if cnt else None,
'url': f"https://www.goodreads.com/book/show/{bid.group(1)}",
})
return {'page': page, 'results': results}
# Example
lst = parse_list("1.Best_Books_Ever")
# lst['results'][0] => {'rank': 1, 'book_id': '2767052',
# 'title': 'The Hunger Games (The Hunger Games, #1)',
# 'author': 'Suzanne Collins', 'avg_rating': 4.35, ...}
```
**Field notes:**
- 100 rows per page. Ranks are sequential across pages (page 2 starts at rank 101).
- Paginate with `?page=2`, `?page=3` etc.
- List pages do not use `__NEXT_DATA__` — same classic HTML format as author pages.
---
## Open Library API Fallback
Use Open Library when you need structured JSON without HTML parsing, or when you want supplementary data (birth/death dates, ISBNs across editions, subjects).
Open Library's ratings are from its own user base (~400 ratings vs. Goodreads' 800k+ for Dune) — use Goodreads ratings when accuracy matters.
### Search
```python
import json
from urllib.parse import quote_plus
from helpers import http_get
def ol_search(query, limit=10):
url = f"https://openlibrary.org/search.json?q={quote_plus(query)}&limit={limit}"
data = json.loads(http_get(url))
results = []
for doc in data.get('docs', []):
cover_id = doc.get('cover_i')
results.append({
'ol_key': doc['key'], # e.g. "/works/OL893415W"
'title': doc.get('title'),
'author': (doc.get('author_name') or [''])[0],
'author_key': (doc.get('author_key') or [''])[0],
'first_pub_year': doc.get('first_publish_year'),
'edition_count': doc.get('edition_count'),
'series': doc.get('series_name'),
'cover_url': f"https://covers.openlibrary.org/b/id/{cover_id}-M.jpg" if cover_id else None,
})
return {'total': data.get('numFound'), 'results': results}
r = ol_search("dune frank herbert", limit=5)
# r['results'][0]['ol_key'] => "/works/OL893415W"
# r['results'][0]['title'] => "Dune"
```
### Work (book details)
```python
def ol_work(ol_key):
# ol_key like "/works/OL893415W" or just "OL893415W"
key = ol_key if ol_key.startswith('/') else f'/works/{ol_key}'
data = json.loads(http_get(f"https://openlibrary.org{key}.json"))
desc = data.get('description', '')
if isinstance(desc, dict):
desc = desc.get('value', '')
return {
'title': data.get('title'),
'subjects': data.get('subjects', []),
'series': data.get('series', []),
'description': desc,
'covers': data.get('covers', []),
'links': data.get('links', []),
}
work = ol_work("OL893415W")
# work['title'] => "Dune"
# work['subjects'] => ["Dune (Imaginary place)", "Fiction", ...]
```
### Ratings for a work
```python
def ol_ratings(ol_key):
key = ol_key if ol_key.startswith('/') else f'/works/{ol_key}'
data = json.loads(http_get(f"https://openlibrary.org{key}/ratings.json"))
return data.get('summary', {})
# {'average': 4.30, 'count': 414, 'sortable': 4.21}
```
### Author
```python
def ol_author(author_key):
# author_key like "OL79034A"
data = json.loads(http_get(f"https://openlibrary.org/authors/{author_key}.json"))
bio = data.get('bio', '')
if isinstance(bio, dict):
bio = bio.get('value', '')
return {
'name': data.get('name'),
'birth_date': data.get('birth_date'),
'death_date': data.get('death_date'),
'bio': bio,
'ol_key': data.get('key'),
}
author = ol_author("OL79034A")
# author['name'] => "Frank Herbert"
# author['birth_date'] => "8 October 1920"
# author['death_date'] => "11 February 1986"
```
---
## Combining Goodreads + Open Library
```python
# Get full book data: Goodreads for ratings/genres/description, OL for ISBNs/edition details
def get_book_full(goodreads_book_id, ol_work_key=None):
gr = parse_book(goodreads_book_id)
result = dict(gr)
if ol_work_key:
ol = ol_work(ol_work_key)
result['ol_subjects'] = ol['subjects']
result['ol_description'] = ol['description']
result['ol_covers'] = ol['covers']
return result
```
---
## Gotchas
- **Goodreads API is gone**: The official API was shut down in December 2020. All data must come from HTML scraping or the unofficial paths documented here.
- **Book ID 5107 redirects**: The URL `goodreads.com/book/show/5107.The_Stand` actually resolves to *The Catcher in the Rye* (ID 5107). The Stand is ID `149267`. Always verify `book['legacyId']` matches the URL ID.
- **Author page ID mismatch**: Author ID `10538` in the URL resolves to Carl Sagan, not Frank Herbert (ID `58`). Always obtain author IDs from the `author_url` field inside a book's data rather than guessing.
- **Two Book entities in `apolloState`**: The `apolloState` contains multiple `Book:` entries — one is a stub (only has `legacyId` and `webUrl`), and one is full. Filter by `legacyId == int(book_id)` AND check that the entry has more than 3 fields.
- **Ratings are on `Work`, not `Book`**: `avg_rating`, `ratingsCount`, and `ratingsCountDist` are in the `Work` entity's `stats` key. The `Book` entity has no rating fields.
- **Author pages are old-style HTML**: Author pages (`/author/show/`) do not use Next.js or `__NEXT_DATA__`. Use OG meta tags and regex for extraction. The follower count only loads via JS — it will be missing from `http_get` responses.
- **Search has no `__NEXT_DATA__`**: Search result pages (`/search`) are classic server-rendered HTML. JSON-LD is absent. Use the `` microdata rows.
- **`ratings_count` regex order matters**: The pattern `r'(\d[\d,]*)\s*rating'` always matches the book's aggregate rating count first in each search row — this is reliable. Do not use `minirating` span text as it contains nested HTML.
- **Open Library cover URLs return binary JPEG**: `http_get()` will raise a `UnicodeDecodeError` on cover image URLs. Use `urllib.request.urlopen()` directly and read bytes, or just store the URL string without fetching.
- **Open Library ratings are sparse**: OL has ~400 community ratings for Dune vs. Goodreads' 1.6M. Use OL ratings only as a last resort.
- **Search page `—` entity**: The raw HTML uses `—` (not `—`) between rating value and count in search and author pages. The regex patterns above match the decoded text because Python's `re` operates on the decoded string after `http_get()` decodes UTF-8.
- **Book slug is optional**: `goodreads.com/book/show/44767458` (no slug) works identically to `goodreads.com/book/show/44767458-dune`. Redirects are transparent.