facebookresearch--audiocraft
Alexandre Défossez (autodoc)
1320 行
72 KiB
HTML
1320 行
72 KiB
HTML
<!doctype html>
|
||
<html lang="en">
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1, minimum-scale=1">
|
||
<meta name="generator" content="pdoc3 0.11.5">
|
||
<title>audiocraft.modules.codebooks_patterns API documentation</title>
|
||
<meta name="description" content="">
|
||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/10up-sanitize.css/13.0.0/sanitize.min.css" integrity="sha512-y1dtMcuvtTMJc1yPgEqF0ZjQbhnc/bFhyvIyVNb9Zk5mIGtqVaAB1Ttl28su8AvFMOY0EwRbAe+HCLqj6W7/KA==" crossorigin>
|
||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/10up-sanitize.css/13.0.0/typography.min.css" integrity="sha512-Y1DYSb995BAfxobCkKepB1BqJJTPrOp3zPL74AWFugHHmmdcvO+C48WLrUOlhGMc0QG7AE3f7gmvvcrmX2fDoA==" crossorigin>
|
||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/default.min.css" crossorigin>
|
||
<style>:root{--highlight-color:#fe9}.flex{display:flex !important}body{line-height:1.5em}#content{padding:20px}#sidebar{padding:1.5em;overflow:hidden}#sidebar > *:last-child{margin-bottom:2cm}.http-server-breadcrumbs{font-size:130%;margin:0 0 15px 0}#footer{font-size:.75em;padding:5px 30px;border-top:1px solid #ddd;text-align:right}#footer p{margin:0 0 0 1em;display:inline-block}#footer p:last-child{margin-right:30px}h1,h2,h3,h4,h5{font-weight:300}h1{font-size:2.5em;line-height:1.1em}h2{font-size:1.75em;margin:2em 0 .50em 0}h3{font-size:1.4em;margin:1.6em 0 .7em 0}h4{margin:0;font-size:105%}h1:target,h2:target,h3:target,h4:target,h5:target,h6:target{background:var(--highlight-color);padding:.2em 0}a{color:#058;text-decoration:none;transition:color .2s ease-in-out}a:visited{color:#503}a:hover{color:#b62}.title code{font-weight:bold}h2[id^="header-"]{margin-top:2em}.ident{color:#900;font-weight:bold}pre code{font-size:.8em;line-height:1.4em;padding:1em;display:block}code{background:#f3f3f3;font-family:"DejaVu Sans Mono",monospace;padding:1px 4px;overflow-wrap:break-word}h1 code{background:transparent}pre{border-top:1px solid #ccc;border-bottom:1px solid #ccc;margin:1em 0}#http-server-module-list{display:flex;flex-flow:column}#http-server-module-list div{display:flex}#http-server-module-list dt{min-width:10%}#http-server-module-list p{margin-top:0}.toc ul,#index{list-style-type:none;margin:0;padding:0}#index code{background:transparent}#index h3{border-bottom:1px solid #ddd}#index ul{padding:0}#index h4{margin-top:.6em;font-weight:bold}@media (min-width:200ex){#index .two-column{column-count:2}}@media (min-width:300ex){#index .two-column{column-count:3}}dl{margin-bottom:2em}dl dl:last-child{margin-bottom:4em}dd{margin:0 0 1em 3em}#header-classes + dl > dd{margin-bottom:3em}dd dd{margin-left:2em}dd p{margin:10px 0}.name{background:#eee;font-size:.85em;padding:5px 10px;display:inline-block;min-width:40%}.name:hover{background:#e0e0e0}dt:target .name{background:var(--highlight-color)}.name > span:first-child{white-space:nowrap}.name.class > span:nth-child(2){margin-left:.4em}.inherited{color:#999;border-left:5px solid #eee;padding-left:1em}.inheritance em{font-style:normal;font-weight:bold}.desc h2{font-weight:400;font-size:1.25em}.desc h3{font-size:1em}.desc dt code{background:inherit}.source > summary,.git-link-div{color:#666;text-align:right;font-weight:400;font-size:.8em;text-transform:uppercase}.source summary > *{white-space:nowrap;cursor:pointer}.git-link{color:inherit;margin-left:1em}.source pre{max-height:500px;overflow:auto;margin:0}.source pre code{font-size:12px;overflow:visible;min-width:max-content}.hlist{list-style:none}.hlist li{display:inline}.hlist li:after{content:',\2002'}.hlist li:last-child:after{content:none}.hlist .hlist{display:inline;padding-left:1em}img{max-width:100%}td{padding:0 .5em}.admonition{padding:.1em 1em;margin:1em 0}.admonition-title{font-weight:bold}.admonition.note,.admonition.info,.admonition.important{background:#aef}.admonition.todo,.admonition.versionadded,.admonition.tip,.admonition.hint{background:#dfd}.admonition.warning,.admonition.versionchanged,.admonition.deprecated{background:#fd4}.admonition.error,.admonition.danger,.admonition.caution{background:lightpink}</style>
|
||
<style media="screen and (min-width: 700px)">@media screen and (min-width:700px){#sidebar{width:30%;height:100vh;overflow:auto;position:sticky;top:0}#content{width:70%;max-width:100ch;padding:3em 4em;border-left:1px solid #ddd}pre code{font-size:1em}.name{font-size:1em}main{display:flex;flex-direction:row-reverse;justify-content:flex-end}.toc ul ul,#index ul ul{padding-left:1em}.toc > ul > li{margin-top:.5em}}</style>
|
||
<style media="print">@media print{#sidebar h1{page-break-before:always}.source{display:none}}@media print{*{background:transparent !important;color:#000 !important;box-shadow:none !important;text-shadow:none !important}a[href]:after{content:" (" attr(href) ")";font-size:90%}a[href][title]:after{content:none}abbr[title]:after{content:" (" attr(title) ")"}.ir a:after,a[href^="javascript:"]:after,a[href^="#"]:after{content:""}pre,blockquote{border:1px solid #999;page-break-inside:avoid}thead{display:table-header-group}tr,img{page-break-inside:avoid}img{max-width:100% !important}@page{margin:0.5cm}p,h2,h3{orphans:3;widows:3}h1,h2,h3,h4,h5,h6{page-break-after:avoid}}</style>
|
||
<script defer src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js" integrity="sha512-D9gUyxqja7hBtkWpPWGt9wfbfaMGVt9gnyCvYa+jojwwPHLCzUm5i8rpk7vD7wNee9bA35eYIjobYPaQuKS1MQ==" crossorigin></script>
|
||
<script>window.addEventListener('DOMContentLoaded', () => {
|
||
hljs.configure({languages: ['bash', 'css', 'diff', 'graphql', 'ini', 'javascript', 'json', 'plaintext', 'python', 'python-repl', 'rust', 'shell', 'sql', 'typescript', 'xml', 'yaml']});
|
||
hljs.highlightAll();
|
||
/* Collapse source docstrings */
|
||
setTimeout(() => {
|
||
[...document.querySelectorAll('.hljs.language-python > .hljs-string')]
|
||
.filter(el => el.innerHTML.length > 200 && ['"""', "'''"].includes(el.innerHTML.substring(0, 3)))
|
||
.forEach(el => {
|
||
let d = document.createElement('details');
|
||
d.classList.add('hljs-string');
|
||
d.innerHTML = '<summary>"""</summary>' + el.innerHTML.substring(3);
|
||
el.replaceWith(d);
|
||
});
|
||
}, 100);
|
||
})</script>
|
||
</head>
|
||
<body>
|
||
<main>
|
||
<article id="content">
|
||
<header>
|
||
<h1 class="title">Module <code>audiocraft.modules.codebooks_patterns</code></h1>
|
||
</header>
|
||
<section id="section-intro">
|
||
</section>
|
||
<section>
|
||
</section>
|
||
<section>
|
||
</section>
|
||
<section>
|
||
</section>
|
||
<section>
|
||
<h2 class="section-title" id="header-classes">Classes</h2>
|
||
<dl>
|
||
<dt id="audiocraft.modules.codebooks_patterns.CoarseFirstPattern"><code class="flex name class">
|
||
<span>class <span class="ident">CoarseFirstPattern</span></span>
|
||
<span>(</span><span>n_q: int, delays: List[int] | None = None)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class CoarseFirstPattern(CodebooksPatternProvider):
|
||
"""First generates all the codebooks #1 (e.g. coarser), then the remaining ones,
|
||
potentially with delays.
|
||
|
||
..Warning:: You must always generate the full training duration at test time, for instance,
|
||
30 seconds, as otherwise, the fine codebooks will start being generated in an unexpected
|
||
location. This is due to the non causality of the remaining codebooks with respect to
|
||
the first ones.
|
||
|
||
Args:
|
||
n_q (int): Number of codebooks.
|
||
delays (list of int, optional): Delay for each of the codebooks.
|
||
If delays not defined, each codebook is delayed by 1 compared to the previous one.
|
||
"""
|
||
def __init__(self, n_q: int, delays: tp.Optional[tp.List[int]] = None):
|
||
super().__init__(n_q)
|
||
if delays is None:
|
||
delays = [0] * (n_q - 1)
|
||
self.delays = delays
|
||
assert len(self.delays) == self.n_q - 1
|
||
assert sorted(self.delays) == self.delays
|
||
|
||
def get_pattern(self, timesteps: int) -> Pattern:
|
||
out: PatternLayout = [[]]
|
||
for t in range(timesteps):
|
||
out.append([LayoutCoord(t, 0)])
|
||
max_delay = max(self.delays)
|
||
for t in range(timesteps + max_delay):
|
||
v = []
|
||
for q, delay in enumerate(self.delays):
|
||
t_for_q = t - delay
|
||
if t_for_q >= 0:
|
||
v.append(LayoutCoord(t_for_q, q + 1))
|
||
out.append(v)
|
||
return Pattern(out, n_q=self.n_q, timesteps=timesteps)</code></pre>
|
||
</details>
|
||
<div class="desc"><p>First generates all the codebooks #1 (e.g. coarser), then the remaining ones,
|
||
potentially with delays.</p>
|
||
<div class="admonition warning">
|
||
<p class="admonition-title">Warning: You must always generate the full training duration at test time, for instance,</p>
|
||
<p>30 seconds, as otherwise, the fine codebooks will start being generated in an unexpected
|
||
location. This is due to the non causality of the remaining codebooks with respect to
|
||
the first ones.</p>
|
||
</div>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>n_q</code></strong> : <code>int</code></dt>
|
||
<dd>Number of codebooks.</dd>
|
||
<dt><strong><code>delays</code></strong> : <code>list</code> of <code>int</code>, optional</dt>
|
||
<dd>Delay for each of the codebooks.
|
||
If delays not defined, each codebook is delayed by 1 compared to the previous one.</dd>
|
||
</dl></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.codebooks_patterns.CodebooksPatternProvider" href="#audiocraft.modules.codebooks_patterns.CodebooksPatternProvider">CodebooksPatternProvider</a></li>
|
||
<li>abc.ABC</li>
|
||
</ul>
|
||
<h3>Inherited members</h3>
|
||
<ul class="hlist">
|
||
<li><code><b><a title="audiocraft.modules.codebooks_patterns.CodebooksPatternProvider" href="#audiocraft.modules.codebooks_patterns.CodebooksPatternProvider">CodebooksPatternProvider</a></b></code>:
|
||
<ul class="hlist">
|
||
<li><code><a title="audiocraft.modules.codebooks_patterns.CodebooksPatternProvider.get_pattern" href="#audiocraft.modules.codebooks_patterns.CodebooksPatternProvider.get_pattern">get_pattern</a></code></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</dd>
|
||
<dt id="audiocraft.modules.codebooks_patterns.CodebooksPatternProvider"><code class="flex name class">
|
||
<span>class <span class="ident">CodebooksPatternProvider</span></span>
|
||
<span>(</span><span>n_q: int, cached: bool = True)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class CodebooksPatternProvider(ABC):
|
||
"""Abstraction around providing pattern for interleaving codebooks.
|
||
|
||
The CodebooksPatternProvider abstraction allows to implement various strategies to
|
||
define interleaving pattern of sequences composed of multiple codebooks. For a given
|
||
number of codebooks `n_q`, the pattern provider can generate a specified pattern
|
||
corresponding to a sequence of `T` timesteps with `n_q` parallel codebooks. This pattern
|
||
can be used to construct a new sequence from the original codes respecting the specified
|
||
pattern. The pattern is defined as a list of list of code coordinates, code coordinate
|
||
being a tuple with the original timestep and codebook to build the new sequence.
|
||
Note that all patterns must start with an empty list that is then used to insert a first
|
||
sequence step of special tokens in the newly generated sequence.
|
||
|
||
Args:
|
||
n_q (int): number of codebooks.
|
||
cached (bool): if True, patterns for a given length are cached. In general
|
||
that should be true for efficiency reason to avoid synchronization points.
|
||
"""
|
||
def __init__(self, n_q: int, cached: bool = True):
|
||
assert n_q > 0
|
||
self.n_q = n_q
|
||
self.get_pattern = lru_cache(100)(self.get_pattern) # type: ignore
|
||
|
||
@abstractmethod
|
||
def get_pattern(self, timesteps: int) -> Pattern:
|
||
"""Builds pattern with specific interleaving between codebooks.
|
||
|
||
Args:
|
||
timesteps (int): Total number of timesteps.
|
||
"""
|
||
raise NotImplementedError()</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Abstraction around providing pattern for interleaving codebooks.</p>
|
||
<p>The CodebooksPatternProvider abstraction allows to implement various strategies to
|
||
define interleaving pattern of sequences composed of multiple codebooks. For a given
|
||
number of codebooks <code>n_q</code>, the pattern provider can generate a specified pattern
|
||
corresponding to a sequence of <code>T</code> timesteps with <code>n_q</code> parallel codebooks. This pattern
|
||
can be used to construct a new sequence from the original codes respecting the specified
|
||
pattern. The pattern is defined as a list of list of code coordinates, code coordinate
|
||
being a tuple with the original timestep and codebook to build the new sequence.
|
||
Note that all patterns must start with an empty list that is then used to insert a first
|
||
sequence step of special tokens in the newly generated sequence.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>n_q</code></strong> : <code>int</code></dt>
|
||
<dd>number of codebooks.</dd>
|
||
<dt><strong><code>cached</code></strong> : <code>bool</code></dt>
|
||
<dd>if True, patterns for a given length are cached. In general
|
||
that should be true for efficiency reason to avoid synchronization points.</dd>
|
||
</dl></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li>abc.ABC</li>
|
||
</ul>
|
||
<h3>Subclasses</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.codebooks_patterns.CoarseFirstPattern" href="#audiocraft.modules.codebooks_patterns.CoarseFirstPattern">CoarseFirstPattern</a></li>
|
||
<li><a title="audiocraft.modules.codebooks_patterns.DelayedPatternProvider" href="#audiocraft.modules.codebooks_patterns.DelayedPatternProvider">DelayedPatternProvider</a></li>
|
||
<li><a title="audiocraft.modules.codebooks_patterns.MusicLMPattern" href="#audiocraft.modules.codebooks_patterns.MusicLMPattern">MusicLMPattern</a></li>
|
||
<li><a title="audiocraft.modules.codebooks_patterns.UnrolledPatternProvider" href="#audiocraft.modules.codebooks_patterns.UnrolledPatternProvider">UnrolledPatternProvider</a></li>
|
||
</ul>
|
||
<h3>Methods</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.codebooks_patterns.CodebooksPatternProvider.get_pattern"><code class="name flex">
|
||
<span>def <span class="ident">get_pattern</span></span>(<span>self, timesteps: int) ‑> <a title="audiocraft.modules.codebooks_patterns.Pattern" href="#audiocraft.modules.codebooks_patterns.Pattern">Pattern</a></span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">@abstractmethod
|
||
def get_pattern(self, timesteps: int) -> Pattern:
|
||
"""Builds pattern with specific interleaving between codebooks.
|
||
|
||
Args:
|
||
timesteps (int): Total number of timesteps.
|
||
"""
|
||
raise NotImplementedError()</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Builds pattern with specific interleaving between codebooks.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>timesteps</code></strong> : <code>int</code></dt>
|
||
<dd>Total number of timesteps.</dd>
|
||
</dl></div>
|
||
</dd>
|
||
</dl>
|
||
</dd>
|
||
<dt id="audiocraft.modules.codebooks_patterns.DelayedPatternProvider"><code class="flex name class">
|
||
<span>class <span class="ident">DelayedPatternProvider</span></span>
|
||
<span>(</span><span>n_q: int,<br>delays: List[int] | None = None,<br>flatten_first: int = 0,<br>empty_initial: int = 0)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class DelayedPatternProvider(CodebooksPatternProvider):
|
||
"""Provider for delayed pattern across delayed codebooks.
|
||
Codebooks are delayed in the sequence and sequence steps will contain codebooks
|
||
from different timesteps.
|
||
|
||
Example:
|
||
Taking timesteps=4 and n_q=3, delays=None, the multi-codebook sequence:
|
||
[[1, 2, 3, 4],
|
||
[1, 2, 3, 4],
|
||
[1, 2, 3, 4]]
|
||
The resulting sequence obtained from the returned pattern is:
|
||
[[S, 1, 2, 3, 4],
|
||
[S, S, 1, 2, 3],
|
||
[S, S, S, 1, 2]]
|
||
(with S being a special token)
|
||
|
||
Args:
|
||
n_q (int): Number of codebooks.
|
||
delays (list of int, optional): Delay for each of the codebooks.
|
||
If delays not defined, each codebook is delayed by 1 compared to the previous one.
|
||
flatten_first (int): Flatten the first N timesteps.
|
||
empty_initial (int): Prepend with N empty list of coordinates.
|
||
"""
|
||
def __init__(self, n_q: int, delays: tp.Optional[tp.List[int]] = None,
|
||
flatten_first: int = 0, empty_initial: int = 0):
|
||
super().__init__(n_q)
|
||
if delays is None:
|
||
delays = list(range(n_q))
|
||
self.delays = delays
|
||
self.flatten_first = flatten_first
|
||
self.empty_initial = empty_initial
|
||
assert len(self.delays) == self.n_q
|
||
assert sorted(self.delays) == self.delays
|
||
|
||
def get_pattern(self, timesteps: int) -> Pattern:
|
||
omit_special_token = self.empty_initial < 0
|
||
out: PatternLayout = [] if omit_special_token else [[]]
|
||
max_delay = max(self.delays)
|
||
if self.empty_initial:
|
||
out += [[] for _ in range(self.empty_initial)]
|
||
if self.flatten_first:
|
||
for t in range(min(timesteps, self.flatten_first)):
|
||
for q in range(self.n_q):
|
||
out.append([LayoutCoord(t, q)])
|
||
for t in range(self.flatten_first, timesteps + max_delay):
|
||
v = []
|
||
for q, delay in enumerate(self.delays):
|
||
t_for_q = t - delay
|
||
if t_for_q >= self.flatten_first:
|
||
v.append(LayoutCoord(t_for_q, q))
|
||
out.append(v)
|
||
return Pattern(out, n_q=self.n_q, timesteps=timesteps)</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Provider for delayed pattern across delayed codebooks.
|
||
Codebooks are delayed in the sequence and sequence steps will contain codebooks
|
||
from different timesteps.</p>
|
||
<h2 id="example">Example</h2>
|
||
<p>Taking timesteps=4 and n_q=3, delays=None, the multi-codebook sequence:
|
||
[[1, 2, 3, 4],
|
||
[1, 2, 3, 4],
|
||
[1, 2, 3, 4]]
|
||
The resulting sequence obtained from the returned pattern is:
|
||
[[S, 1, 2, 3, 4],
|
||
[S, S, 1, 2, 3],
|
||
[S, S, S, 1, 2]]
|
||
(with S being a special token)</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>n_q</code></strong> : <code>int</code></dt>
|
||
<dd>Number of codebooks.</dd>
|
||
<dt><strong><code>delays</code></strong> : <code>list</code> of <code>int</code>, optional</dt>
|
||
<dd>Delay for each of the codebooks.
|
||
If delays not defined, each codebook is delayed by 1 compared to the previous one.</dd>
|
||
<dt><strong><code>flatten_first</code></strong> : <code>int</code></dt>
|
||
<dd>Flatten the first N timesteps.</dd>
|
||
<dt><strong><code>empty_initial</code></strong> : <code>int</code></dt>
|
||
<dd>Prepend with N empty list of coordinates.</dd>
|
||
</dl></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.codebooks_patterns.CodebooksPatternProvider" href="#audiocraft.modules.codebooks_patterns.CodebooksPatternProvider">CodebooksPatternProvider</a></li>
|
||
<li>abc.ABC</li>
|
||
</ul>
|
||
<h3>Subclasses</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.codebooks_patterns.ParallelPatternProvider" href="#audiocraft.modules.codebooks_patterns.ParallelPatternProvider">ParallelPatternProvider</a></li>
|
||
</ul>
|
||
<h3>Inherited members</h3>
|
||
<ul class="hlist">
|
||
<li><code><b><a title="audiocraft.modules.codebooks_patterns.CodebooksPatternProvider" href="#audiocraft.modules.codebooks_patterns.CodebooksPatternProvider">CodebooksPatternProvider</a></b></code>:
|
||
<ul class="hlist">
|
||
<li><code><a title="audiocraft.modules.codebooks_patterns.CodebooksPatternProvider.get_pattern" href="#audiocraft.modules.codebooks_patterns.CodebooksPatternProvider.get_pattern">get_pattern</a></code></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</dd>
|
||
<dt id="audiocraft.modules.codebooks_patterns.LayoutCoord"><code class="flex name class">
|
||
<span>class <span class="ident">LayoutCoord</span></span>
|
||
<span>(</span><span>t, q)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<div class="desc"><p>LayoutCoord(t, q)</p></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li>builtins.tuple</li>
|
||
</ul>
|
||
<h3>Instance variables</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.codebooks_patterns.LayoutCoord.q"><code class="name">var <span class="ident">q</span></code></dt>
|
||
<dd>
|
||
<div class="desc"><p>Alias for field number 1</p></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.codebooks_patterns.LayoutCoord.t"><code class="name">var <span class="ident">t</span></code></dt>
|
||
<dd>
|
||
<div class="desc"><p>Alias for field number 0</p></div>
|
||
</dd>
|
||
</dl>
|
||
</dd>
|
||
<dt id="audiocraft.modules.codebooks_patterns.MusicLMPattern"><code class="flex name class">
|
||
<span>class <span class="ident">MusicLMPattern</span></span>
|
||
<span>(</span><span>n_q: int, group_by: int = 2)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class MusicLMPattern(CodebooksPatternProvider):
|
||
"""Almost MusicLM style pattern. This is equivalent to full flattening
|
||
but in a different order.
|
||
|
||
Args:
|
||
n_q (int): Number of codebooks.
|
||
group_by (int): Number of codebooks to group together.
|
||
"""
|
||
def __init__(self, n_q: int, group_by: int = 2):
|
||
super().__init__(n_q)
|
||
self.group_by = group_by
|
||
|
||
def get_pattern(self, timesteps: int) -> Pattern:
|
||
out: PatternLayout = [[]]
|
||
for offset in range(0, self.n_q, self.group_by):
|
||
for t in range(timesteps):
|
||
for q in range(offset, offset + self.group_by):
|
||
out.append([LayoutCoord(t, q)])
|
||
return Pattern(out, n_q=self.n_q, timesteps=timesteps)</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Almost MusicLM style pattern. This is equivalent to full flattening
|
||
but in a different order.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>n_q</code></strong> : <code>int</code></dt>
|
||
<dd>Number of codebooks.</dd>
|
||
<dt><strong><code>group_by</code></strong> : <code>int</code></dt>
|
||
<dd>Number of codebooks to group together.</dd>
|
||
</dl></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.codebooks_patterns.CodebooksPatternProvider" href="#audiocraft.modules.codebooks_patterns.CodebooksPatternProvider">CodebooksPatternProvider</a></li>
|
||
<li>abc.ABC</li>
|
||
</ul>
|
||
<h3>Inherited members</h3>
|
||
<ul class="hlist">
|
||
<li><code><b><a title="audiocraft.modules.codebooks_patterns.CodebooksPatternProvider" href="#audiocraft.modules.codebooks_patterns.CodebooksPatternProvider">CodebooksPatternProvider</a></b></code>:
|
||
<ul class="hlist">
|
||
<li><code><a title="audiocraft.modules.codebooks_patterns.CodebooksPatternProvider.get_pattern" href="#audiocraft.modules.codebooks_patterns.CodebooksPatternProvider.get_pattern">get_pattern</a></code></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</dd>
|
||
<dt id="audiocraft.modules.codebooks_patterns.ParallelPatternProvider"><code class="flex name class">
|
||
<span>class <span class="ident">ParallelPatternProvider</span></span>
|
||
<span>(</span><span>n_q: int, empty_initial: int = 0)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class ParallelPatternProvider(DelayedPatternProvider):
|
||
"""Provider for parallel pattern across codebooks.
|
||
This pattern provider is a special case of the delayed pattern with actually no delay,
|
||
hence delays=repeat(0, n_q).
|
||
|
||
Args:
|
||
n_q (int): Number of codebooks.
|
||
empty_initial (int): Prepend with N empty list of coordinates.
|
||
"""
|
||
def __init__(self, n_q: int, empty_initial: int = 0):
|
||
super().__init__(n_q, [0] * n_q, empty_initial=empty_initial)</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Provider for parallel pattern across codebooks.
|
||
This pattern provider is a special case of the delayed pattern with actually no delay,
|
||
hence delays=repeat(0, n_q).</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>n_q</code></strong> : <code>int</code></dt>
|
||
<dd>Number of codebooks.</dd>
|
||
<dt><strong><code>empty_initial</code></strong> : <code>int</code></dt>
|
||
<dd>Prepend with N empty list of coordinates.</dd>
|
||
</dl></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.codebooks_patterns.DelayedPatternProvider" href="#audiocraft.modules.codebooks_patterns.DelayedPatternProvider">DelayedPatternProvider</a></li>
|
||
<li><a title="audiocraft.modules.codebooks_patterns.CodebooksPatternProvider" href="#audiocraft.modules.codebooks_patterns.CodebooksPatternProvider">CodebooksPatternProvider</a></li>
|
||
<li>abc.ABC</li>
|
||
</ul>
|
||
<h3>Inherited members</h3>
|
||
<ul class="hlist">
|
||
<li><code><b><a title="audiocraft.modules.codebooks_patterns.DelayedPatternProvider" href="#audiocraft.modules.codebooks_patterns.DelayedPatternProvider">DelayedPatternProvider</a></b></code>:
|
||
<ul class="hlist">
|
||
<li><code><a title="audiocraft.modules.codebooks_patterns.DelayedPatternProvider.get_pattern" href="#audiocraft.modules.codebooks_patterns.CodebooksPatternProvider.get_pattern">get_pattern</a></code></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</dd>
|
||
<dt id="audiocraft.modules.codebooks_patterns.Pattern"><code class="flex name class">
|
||
<span>class <span class="ident">Pattern</span></span>
|
||
<span>(</span><span>layout: List[List[<a title="audiocraft.modules.codebooks_patterns.LayoutCoord" href="#audiocraft.modules.codebooks_patterns.LayoutCoord">LayoutCoord</a>]],<br>timesteps: int,<br>n_q: int)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">@dataclass
|
||
class Pattern:
|
||
"""Base implementation of a pattern over a sequence with multiple codebooks.
|
||
|
||
The codebook pattern consists in a layout, defining for each sequence step
|
||
the list of coordinates of each codebook timestep in the resulting interleaved sequence.
|
||
The first item of the pattern is always an empty list in order to properly insert a special token
|
||
to start with. For convenience, we also keep track of ``n_q`` the number of codebooks used for the pattern
|
||
and ``timesteps`` the number of timesteps corresponding to the original sequence.
|
||
|
||
The pattern provides convenient methods to build and revert interleaved sequences from it:
|
||
``build_pattern_sequence`` maps a given a dense input tensor of multi-codebook sequence from [B, K, T]
|
||
to the interleaved sequence of shape [B, K, S] applying the pattern, with B being the batch size,
|
||
K being the number of codebooks, T the number of original timesteps and S the number of sequence steps
|
||
for the output sequence. The unfilled positions are replaced with a special token and the built sequence
|
||
is returned along with a mask indicating valid tokens.
|
||
``revert_pattern_sequence`` maps back an interleaved sequence of shape [B, K, S] to the original alignment
|
||
of codebooks across timesteps to an output tensor of shape [B, K, T], using again a special token and a mask
|
||
to fill and specify invalid positions if needed.
|
||
See the dedicated methods for more details.
|
||
"""
|
||
# Pattern layout, for each sequence step, we have a list of coordinates
|
||
# corresponding to the original codebook timestep and position.
|
||
# The first list is always an empty list in order to properly insert
|
||
# a special token to start with.
|
||
layout: PatternLayout
|
||
timesteps: int
|
||
n_q: int
|
||
|
||
def __post_init__(self):
|
||
assert len(self.layout) > 0
|
||
self._validate_layout()
|
||
self._build_reverted_sequence_scatter_indexes = lru_cache(100)(self._build_reverted_sequence_scatter_indexes)
|
||
self._build_pattern_sequence_scatter_indexes = lru_cache(100)(self._build_pattern_sequence_scatter_indexes)
|
||
logger.info("New pattern, time steps: %d, sequence steps: %d", self.timesteps, len(self.layout))
|
||
|
||
def _validate_layout(self):
|
||
"""Runs checks on the layout to ensure a valid pattern is defined.
|
||
A pattern is considered invalid if:
|
||
- Multiple timesteps for a same codebook are defined in the same sequence step
|
||
- The timesteps for a given codebook are not in ascending order as we advance in the sequence
|
||
(this would mean that we have future timesteps before past timesteps).
|
||
"""
|
||
q_timesteps = {q: 0 for q in range(self.n_q)}
|
||
for s, seq_coords in enumerate(self.layout):
|
||
if len(seq_coords) > 0:
|
||
qs = set()
|
||
for coord in seq_coords:
|
||
qs.add(coord.q)
|
||
last_q_timestep = q_timesteps[coord.q]
|
||
assert coord.t >= last_q_timestep, \
|
||
f"Past timesteps are found in the sequence for codebook = {coord.q} at step {s}"
|
||
q_timesteps[coord.q] = coord.t
|
||
# each sequence step contains at max 1 coordinate per codebook
|
||
assert len(qs) == len(seq_coords), \
|
||
f"Multiple entries for a same codebook are found at step {s}"
|
||
|
||
@property
|
||
def num_sequence_steps(self):
|
||
return len(self.layout) - 1
|
||
|
||
@property
|
||
def max_delay(self):
|
||
max_t_in_seq_coords = 0
|
||
for seq_coords in self.layout[1:]:
|
||
for coords in seq_coords:
|
||
max_t_in_seq_coords = max(max_t_in_seq_coords, coords.t + 1)
|
||
return max_t_in_seq_coords - self.timesteps
|
||
|
||
@property
|
||
def valid_layout(self):
|
||
valid_step = len(self.layout) - self.max_delay
|
||
return self.layout[:valid_step]
|
||
|
||
def starts_with_special_token(self):
|
||
return self.layout[0] == []
|
||
|
||
def get_sequence_coords_with_timestep(self, t: int, q: tp.Optional[int] = None):
|
||
"""Get codebook coordinates in the layout that corresponds to the specified timestep t
|
||
and optionally to the codebook q. Coordinates are returned as a tuple with the sequence step
|
||
and the actual codebook coordinates.
|
||
"""
|
||
assert t <= self.timesteps, "provided timesteps is greater than the pattern's number of timesteps"
|
||
if q is not None:
|
||
assert q <= self.n_q, "provided number of codebooks is greater than the pattern's number of codebooks"
|
||
coords = []
|
||
for s, seq_codes in enumerate(self.layout):
|
||
for code in seq_codes:
|
||
if code.t == t and (q is None or code.q == q):
|
||
coords.append((s, code))
|
||
return coords
|
||
|
||
def get_steps_with_timestep(self, t: int, q: tp.Optional[int] = None) -> tp.List[int]:
|
||
return [step for step, coords in self.get_sequence_coords_with_timestep(t, q)]
|
||
|
||
def get_first_step_with_timesteps(self, t: int, q: tp.Optional[int] = None) -> tp.Optional[int]:
|
||
steps_with_timesteps = self.get_steps_with_timestep(t, q)
|
||
return steps_with_timesteps[0] if len(steps_with_timesteps) > 0 else None
|
||
|
||
def _build_pattern_sequence_scatter_indexes(self, timesteps: int, n_q: int, keep_only_valid_steps: bool,
|
||
device: tp.Union[torch.device, str] = 'cpu'):
|
||
"""Build scatter indexes corresponding to the pattern, up to the provided sequence_steps.
|
||
|
||
Args:
|
||
timesteps (int): Maximum number of timesteps steps to consider.
|
||
keep_only_valid_steps (bool): Restrict the pattern layout to match only valid steps.
|
||
device (torch.device or str): Device for created tensors.
|
||
Returns:
|
||
indexes (torch.Tensor): Indexes corresponding to the sequence, of shape [K, S].
|
||
mask (torch.Tensor): Mask corresponding to indexes that matches valid indexes, of shape [K, S].
|
||
"""
|
||
assert n_q == self.n_q, f"invalid number of codebooks for the sequence and the pattern: {n_q} != {self.n_q}"
|
||
assert timesteps <= self.timesteps, "invalid number of timesteps used to build the sequence from the pattern"
|
||
# use the proper layout based on whether we limit ourselves to valid steps only or not,
|
||
# note that using the valid_layout will result in a truncated sequence up to the valid steps
|
||
ref_layout = self.valid_layout if keep_only_valid_steps else self.layout
|
||
# single item indexing being super slow with pytorch vs. numpy, so we use numpy here
|
||
indexes = torch.zeros(n_q, len(ref_layout), dtype=torch.long).numpy()
|
||
mask = torch.zeros(n_q, len(ref_layout), dtype=torch.bool).numpy()
|
||
# fill indexes with last sequence step value that will correspond to our special token
|
||
# the last value is n_q * timesteps as we have flattened z and append special token as the last token
|
||
# which will correspond to the index: n_q * timesteps
|
||
indexes[:] = n_q * timesteps
|
||
# iterate over the pattern and fill scattered indexes and mask
|
||
for s, sequence_coords in enumerate(ref_layout):
|
||
for coords in sequence_coords:
|
||
if coords.t < timesteps:
|
||
indexes[coords.q, s] = coords.t + coords.q * timesteps
|
||
mask[coords.q, s] = 1
|
||
indexes = torch.from_numpy(indexes).to(device)
|
||
mask = torch.from_numpy(mask).to(device)
|
||
return indexes, mask
|
||
|
||
def build_pattern_sequence(self, z: torch.Tensor, special_token: int, keep_only_valid_steps: bool = False):
|
||
"""Build sequence corresponding to the pattern from the input tensor z.
|
||
The sequence is built using up to sequence_steps if specified, and non-pattern
|
||
coordinates are filled with the special token.
|
||
|
||
Args:
|
||
z (torch.Tensor): Input tensor of multi-codebooks sequence, of shape [B, K, T].
|
||
special_token (int): Special token used to fill non-pattern coordinates in the new sequence.
|
||
keep_only_valid_steps (bool): Build a sequence from the pattern up to valid (= fully defined) steps.
|
||
Steps that are beyond valid steps will be replaced by the special_token in that case.
|
||
Returns:
|
||
values (torch.Tensor): Interleaved sequence matching the pattern, of shape [B, K, S] with S
|
||
corresponding either to the sequence_steps if provided, otherwise to the length of the pattern.
|
||
indexes (torch.Tensor): Indexes corresponding to the interleaved sequence, of shape [K, S].
|
||
mask (torch.Tensor): Mask corresponding to indexes that matches valid indexes of shape [K, S].
|
||
"""
|
||
B, K, T = z.shape
|
||
indexes, mask = self._build_pattern_sequence_scatter_indexes(
|
||
T, K, keep_only_valid_steps=keep_only_valid_steps, device=str(z.device)
|
||
)
|
||
z = z.view(B, -1)
|
||
# we append the special token as the last index of our flattened z tensor
|
||
z = torch.cat([z, torch.zeros_like(z[:, :1]) + special_token], dim=1)
|
||
values = z[:, indexes.view(-1)]
|
||
values = values.view(B, K, indexes.shape[-1])
|
||
return values, indexes, mask
|
||
|
||
def _build_reverted_sequence_scatter_indexes(self, sequence_steps: int, n_q: int,
|
||
keep_only_valid_steps: bool = False,
|
||
is_model_output: bool = False,
|
||
device: tp.Union[torch.device, str] = 'cpu'):
|
||
"""Builds scatter indexes required to retrieve the original multi-codebook sequence
|
||
from interleaving pattern.
|
||
|
||
Args:
|
||
sequence_steps (int): Sequence steps.
|
||
n_q (int): Number of codebooks.
|
||
keep_only_valid_steps (bool): Build a sequence from the pattern up to valid (= fully defined) steps.
|
||
Steps that are beyond valid steps will be replaced by the special_token in that case.
|
||
is_model_output (bool): Whether to keep the sequence item corresponding to initial special token or not.
|
||
device (torch.device or str): Device for created tensors.
|
||
Returns:
|
||
indexes (torch.Tensor): Indexes for reconstructing the output, of shape [K, T].
|
||
mask (torch.Tensor): Mask corresponding to indexes that matches valid indexes of shape [K, T].
|
||
"""
|
||
ref_layout = self.valid_layout if keep_only_valid_steps else self.layout
|
||
# TODO(jade): Do we want to further truncate to only valid timesteps here as well?
|
||
timesteps = self.timesteps
|
||
assert n_q == self.n_q, f"invalid number of codebooks for the sequence and the pattern: {n_q} != {self.n_q}"
|
||
assert sequence_steps <= len(ref_layout), \
|
||
f"sequence to revert is longer than the defined pattern: {sequence_steps} > {len(ref_layout)}"
|
||
|
||
# ensure we take the appropriate indexes to keep the model output from the first special token as well
|
||
if is_model_output and self.starts_with_special_token():
|
||
ref_layout = ref_layout[1:]
|
||
|
||
# single item indexing being super slow with pytorch vs. numpy, so we use numpy here
|
||
indexes = torch.zeros(n_q, timesteps, dtype=torch.long).numpy()
|
||
mask = torch.zeros(n_q, timesteps, dtype=torch.bool).numpy()
|
||
# fill indexes with last sequence step value that will correspond to our special token
|
||
indexes[:] = n_q * sequence_steps
|
||
for s, sequence_codes in enumerate(ref_layout):
|
||
if s < sequence_steps:
|
||
for code in sequence_codes:
|
||
if code.t < timesteps:
|
||
indexes[code.q, code.t] = s + code.q * sequence_steps
|
||
mask[code.q, code.t] = 1
|
||
indexes = torch.from_numpy(indexes).to(device)
|
||
mask = torch.from_numpy(mask).to(device)
|
||
return indexes, mask
|
||
|
||
def revert_pattern_sequence(self, s: torch.Tensor, special_token: int, keep_only_valid_steps: bool = False):
|
||
"""Revert a sequence built from the pattern back to the original multi-codebook sequence without interleaving.
|
||
The sequence is reverted using up to timesteps if specified, and non-pattern coordinates
|
||
are filled with the special token.
|
||
|
||
Args:
|
||
s (torch.Tensor): Interleaved sequence tensor obtained from the pattern, of shape [B, K, S].
|
||
special_token (int or float): Special token used to fill non-pattern coordinates in the new sequence.
|
||
Returns:
|
||
values (torch.Tensor): Interleaved sequence matching the pattern, of shape [B, K, T] with T
|
||
corresponding either to the timesteps if provided, or the total timesteps in pattern otherwise.
|
||
indexes (torch.Tensor): Indexes corresponding to the interleaved sequence, of shape [K, T].
|
||
mask (torch.Tensor): Mask corresponding to indexes that matches valid indexes of shape [K, T].
|
||
"""
|
||
B, K, S = s.shape
|
||
indexes, mask = self._build_reverted_sequence_scatter_indexes(
|
||
S, K, keep_only_valid_steps, is_model_output=False, device=str(s.device)
|
||
)
|
||
s = s.view(B, -1)
|
||
# we append the special token as the last index of our flattened z tensor
|
||
s = torch.cat([s, torch.zeros_like(s[:, :1]) + special_token], dim=1)
|
||
values = s[:, indexes.view(-1)]
|
||
values = values.view(B, K, indexes.shape[-1])
|
||
return values, indexes, mask
|
||
|
||
def revert_pattern_logits(self, logits: torch.Tensor, special_token: float, keep_only_valid_steps: bool = False):
|
||
"""Revert model logits obtained on a sequence built from the pattern
|
||
back to a tensor matching the original sequence.
|
||
|
||
This method is similar to ``revert_pattern_sequence`` with the following specificities:
|
||
1. It is designed to work with the extra cardinality dimension
|
||
2. We return the logits for the first sequence item that matches the special_token and
|
||
which matching target in the original sequence is the first item of the sequence,
|
||
while we skip the last logits as there is no matching target
|
||
"""
|
||
B, card, K, S = logits.shape
|
||
indexes, mask = self._build_reverted_sequence_scatter_indexes(
|
||
S, K, keep_only_valid_steps, is_model_output=True, device=logits.device
|
||
)
|
||
logits = logits.reshape(B, card, -1)
|
||
# we append the special token as the last index of our flattened z tensor
|
||
logits = torch.cat([logits, torch.zeros_like(logits[:, :, :1]) + special_token], dim=-1) # [B, card, K x S]
|
||
values = logits[:, :, indexes.view(-1)]
|
||
values = values.view(B, card, K, indexes.shape[-1])
|
||
return values, indexes, mask</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Base implementation of a pattern over a sequence with multiple codebooks.</p>
|
||
<p>The codebook pattern consists in a layout, defining for each sequence step
|
||
the list of coordinates of each codebook timestep in the resulting interleaved sequence.
|
||
The first item of the pattern is always an empty list in order to properly insert a special token
|
||
to start with. For convenience, we also keep track of <code>n_q</code> the number of codebooks used for the pattern
|
||
and <code>timesteps</code> the number of timesteps corresponding to the original sequence.</p>
|
||
<p>The pattern provides convenient methods to build and revert interleaved sequences from it:
|
||
<code>build_pattern_sequence</code> maps a given a dense input tensor of multi-codebook sequence from [B, K, T]
|
||
to the interleaved sequence of shape [B, K, S] applying the pattern, with B being the batch size,
|
||
K being the number of codebooks, T the number of original timesteps and S the number of sequence steps
|
||
for the output sequence. The unfilled positions are replaced with a special token and the built sequence
|
||
is returned along with a mask indicating valid tokens.
|
||
<code>revert_pattern_sequence</code> maps back an interleaved sequence of shape [B, K, S] to the original alignment
|
||
of codebooks across timesteps to an output tensor of shape [B, K, T], using again a special token and a mask
|
||
to fill and specify invalid positions if needed.
|
||
See the dedicated methods for more details.</p></div>
|
||
<h3>Class variables</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.codebooks_patterns.Pattern.layout"><code class="name">var <span class="ident">layout</span> : List[List[<a title="audiocraft.modules.codebooks_patterns.LayoutCoord" href="#audiocraft.modules.codebooks_patterns.LayoutCoord">LayoutCoord</a>]]</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.codebooks_patterns.Pattern.n_q"><code class="name">var <span class="ident">n_q</span> : int</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.codebooks_patterns.Pattern.timesteps"><code class="name">var <span class="ident">timesteps</span> : int</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
<h3>Instance variables</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.codebooks_patterns.Pattern.max_delay"><code class="name">prop <span class="ident">max_delay</span></code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">@property
|
||
def max_delay(self):
|
||
max_t_in_seq_coords = 0
|
||
for seq_coords in self.layout[1:]:
|
||
for coords in seq_coords:
|
||
max_t_in_seq_coords = max(max_t_in_seq_coords, coords.t + 1)
|
||
return max_t_in_seq_coords - self.timesteps</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.codebooks_patterns.Pattern.num_sequence_steps"><code class="name">prop <span class="ident">num_sequence_steps</span></code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">@property
|
||
def num_sequence_steps(self):
|
||
return len(self.layout) - 1</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.codebooks_patterns.Pattern.valid_layout"><code class="name">prop <span class="ident">valid_layout</span></code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">@property
|
||
def valid_layout(self):
|
||
valid_step = len(self.layout) - self.max_delay
|
||
return self.layout[:valid_step]</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
<h3>Methods</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.codebooks_patterns.Pattern.build_pattern_sequence"><code class="name flex">
|
||
<span>def <span class="ident">build_pattern_sequence</span></span>(<span>self, z: torch.Tensor, special_token: int, keep_only_valid_steps: bool = False)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def build_pattern_sequence(self, z: torch.Tensor, special_token: int, keep_only_valid_steps: bool = False):
|
||
"""Build sequence corresponding to the pattern from the input tensor z.
|
||
The sequence is built using up to sequence_steps if specified, and non-pattern
|
||
coordinates are filled with the special token.
|
||
|
||
Args:
|
||
z (torch.Tensor): Input tensor of multi-codebooks sequence, of shape [B, K, T].
|
||
special_token (int): Special token used to fill non-pattern coordinates in the new sequence.
|
||
keep_only_valid_steps (bool): Build a sequence from the pattern up to valid (= fully defined) steps.
|
||
Steps that are beyond valid steps will be replaced by the special_token in that case.
|
||
Returns:
|
||
values (torch.Tensor): Interleaved sequence matching the pattern, of shape [B, K, S] with S
|
||
corresponding either to the sequence_steps if provided, otherwise to the length of the pattern.
|
||
indexes (torch.Tensor): Indexes corresponding to the interleaved sequence, of shape [K, S].
|
||
mask (torch.Tensor): Mask corresponding to indexes that matches valid indexes of shape [K, S].
|
||
"""
|
||
B, K, T = z.shape
|
||
indexes, mask = self._build_pattern_sequence_scatter_indexes(
|
||
T, K, keep_only_valid_steps=keep_only_valid_steps, device=str(z.device)
|
||
)
|
||
z = z.view(B, -1)
|
||
# we append the special token as the last index of our flattened z tensor
|
||
z = torch.cat([z, torch.zeros_like(z[:, :1]) + special_token], dim=1)
|
||
values = z[:, indexes.view(-1)]
|
||
values = values.view(B, K, indexes.shape[-1])
|
||
return values, indexes, mask</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Build sequence corresponding to the pattern from the input tensor z.
|
||
The sequence is built using up to sequence_steps if specified, and non-pattern
|
||
coordinates are filled with the special token.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>z</code></strong> : <code>torch.Tensor</code></dt>
|
||
<dd>Input tensor of multi-codebooks sequence, of shape [B, K, T].</dd>
|
||
<dt><strong><code>special_token</code></strong> : <code>int</code></dt>
|
||
<dd>Special token used to fill non-pattern coordinates in the new sequence.</dd>
|
||
<dt><strong><code>keep_only_valid_steps</code></strong> : <code>bool</code></dt>
|
||
<dd>Build a sequence from the pattern up to valid (= fully defined) steps.
|
||
Steps that are beyond valid steps will be replaced by the special_token in that case.</dd>
|
||
</dl>
|
||
<h2 id="returns">Returns</h2>
|
||
<p>values (torch.Tensor): Interleaved sequence matching the pattern, of shape [B, K, S] with S
|
||
corresponding either to the sequence_steps if provided, otherwise to the length of the pattern.
|
||
indexes (torch.Tensor): Indexes corresponding to the interleaved sequence, of shape [K, S].
|
||
mask (torch.Tensor): Mask corresponding to indexes that matches valid indexes of shape [K, S].</p></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.codebooks_patterns.Pattern.get_first_step_with_timesteps"><code class="name flex">
|
||
<span>def <span class="ident">get_first_step_with_timesteps</span></span>(<span>self, t: int, q: int | None = None) ‑> int | None</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def get_first_step_with_timesteps(self, t: int, q: tp.Optional[int] = None) -> tp.Optional[int]:
|
||
steps_with_timesteps = self.get_steps_with_timestep(t, q)
|
||
return steps_with_timesteps[0] if len(steps_with_timesteps) > 0 else None</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.codebooks_patterns.Pattern.get_sequence_coords_with_timestep"><code class="name flex">
|
||
<span>def <span class="ident">get_sequence_coords_with_timestep</span></span>(<span>self, t: int, q: int | None = None)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def get_sequence_coords_with_timestep(self, t: int, q: tp.Optional[int] = None):
|
||
"""Get codebook coordinates in the layout that corresponds to the specified timestep t
|
||
and optionally to the codebook q. Coordinates are returned as a tuple with the sequence step
|
||
and the actual codebook coordinates.
|
||
"""
|
||
assert t <= self.timesteps, "provided timesteps is greater than the pattern's number of timesteps"
|
||
if q is not None:
|
||
assert q <= self.n_q, "provided number of codebooks is greater than the pattern's number of codebooks"
|
||
coords = []
|
||
for s, seq_codes in enumerate(self.layout):
|
||
for code in seq_codes:
|
||
if code.t == t and (q is None or code.q == q):
|
||
coords.append((s, code))
|
||
return coords</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Get codebook coordinates in the layout that corresponds to the specified timestep t
|
||
and optionally to the codebook q. Coordinates are returned as a tuple with the sequence step
|
||
and the actual codebook coordinates.</p></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.codebooks_patterns.Pattern.get_steps_with_timestep"><code class="name flex">
|
||
<span>def <span class="ident">get_steps_with_timestep</span></span>(<span>self, t: int, q: int | None = None) ‑> List[int]</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def get_steps_with_timestep(self, t: int, q: tp.Optional[int] = None) -> tp.List[int]:
|
||
return [step for step, coords in self.get_sequence_coords_with_timestep(t, q)]</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.codebooks_patterns.Pattern.revert_pattern_logits"><code class="name flex">
|
||
<span>def <span class="ident">revert_pattern_logits</span></span>(<span>self,<br>logits: torch.Tensor,<br>special_token: float,<br>keep_only_valid_steps: bool = False)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def revert_pattern_logits(self, logits: torch.Tensor, special_token: float, keep_only_valid_steps: bool = False):
|
||
"""Revert model logits obtained on a sequence built from the pattern
|
||
back to a tensor matching the original sequence.
|
||
|
||
This method is similar to ``revert_pattern_sequence`` with the following specificities:
|
||
1. It is designed to work with the extra cardinality dimension
|
||
2. We return the logits for the first sequence item that matches the special_token and
|
||
which matching target in the original sequence is the first item of the sequence,
|
||
while we skip the last logits as there is no matching target
|
||
"""
|
||
B, card, K, S = logits.shape
|
||
indexes, mask = self._build_reverted_sequence_scatter_indexes(
|
||
S, K, keep_only_valid_steps, is_model_output=True, device=logits.device
|
||
)
|
||
logits = logits.reshape(B, card, -1)
|
||
# we append the special token as the last index of our flattened z tensor
|
||
logits = torch.cat([logits, torch.zeros_like(logits[:, :, :1]) + special_token], dim=-1) # [B, card, K x S]
|
||
values = logits[:, :, indexes.view(-1)]
|
||
values = values.view(B, card, K, indexes.shape[-1])
|
||
return values, indexes, mask</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Revert model logits obtained on a sequence built from the pattern
|
||
back to a tensor matching the original sequence.</p>
|
||
<p>This method is similar to <code>revert_pattern_sequence</code> with the following specificities:
|
||
1. It is designed to work with the extra cardinality dimension
|
||
2. We return the logits for the first sequence item that matches the special_token and
|
||
which matching target in the original sequence is the first item of the sequence,
|
||
while we skip the last logits as there is no matching target</p></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.codebooks_patterns.Pattern.revert_pattern_sequence"><code class="name flex">
|
||
<span>def <span class="ident">revert_pattern_sequence</span></span>(<span>self, s: torch.Tensor, special_token: int, keep_only_valid_steps: bool = False)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def revert_pattern_sequence(self, s: torch.Tensor, special_token: int, keep_only_valid_steps: bool = False):
|
||
"""Revert a sequence built from the pattern back to the original multi-codebook sequence without interleaving.
|
||
The sequence is reverted using up to timesteps if specified, and non-pattern coordinates
|
||
are filled with the special token.
|
||
|
||
Args:
|
||
s (torch.Tensor): Interleaved sequence tensor obtained from the pattern, of shape [B, K, S].
|
||
special_token (int or float): Special token used to fill non-pattern coordinates in the new sequence.
|
||
Returns:
|
||
values (torch.Tensor): Interleaved sequence matching the pattern, of shape [B, K, T] with T
|
||
corresponding either to the timesteps if provided, or the total timesteps in pattern otherwise.
|
||
indexes (torch.Tensor): Indexes corresponding to the interleaved sequence, of shape [K, T].
|
||
mask (torch.Tensor): Mask corresponding to indexes that matches valid indexes of shape [K, T].
|
||
"""
|
||
B, K, S = s.shape
|
||
indexes, mask = self._build_reverted_sequence_scatter_indexes(
|
||
S, K, keep_only_valid_steps, is_model_output=False, device=str(s.device)
|
||
)
|
||
s = s.view(B, -1)
|
||
# we append the special token as the last index of our flattened z tensor
|
||
s = torch.cat([s, torch.zeros_like(s[:, :1]) + special_token], dim=1)
|
||
values = s[:, indexes.view(-1)]
|
||
values = values.view(B, K, indexes.shape[-1])
|
||
return values, indexes, mask</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Revert a sequence built from the pattern back to the original multi-codebook sequence without interleaving.
|
||
The sequence is reverted using up to timesteps if specified, and non-pattern coordinates
|
||
are filled with the special token.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>s</code></strong> : <code>torch.Tensor</code></dt>
|
||
<dd>Interleaved sequence tensor obtained from the pattern, of shape [B, K, S].</dd>
|
||
<dt><strong><code>special_token</code></strong> : <code>int</code> or <code>float</code></dt>
|
||
<dd>Special token used to fill non-pattern coordinates in the new sequence.</dd>
|
||
</dl>
|
||
<h2 id="returns">Returns</h2>
|
||
<p>values (torch.Tensor): Interleaved sequence matching the pattern, of shape [B, K, T] with T
|
||
corresponding either to the timesteps if provided, or the total timesteps in pattern otherwise.
|
||
indexes (torch.Tensor): Indexes corresponding to the interleaved sequence, of shape [K, T].
|
||
mask (torch.Tensor): Mask corresponding to indexes that matches valid indexes of shape [K, T].</p></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.codebooks_patterns.Pattern.starts_with_special_token"><code class="name flex">
|
||
<span>def <span class="ident">starts_with_special_token</span></span>(<span>self)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def starts_with_special_token(self):
|
||
return self.layout[0] == []</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
</dd>
|
||
<dt id="audiocraft.modules.codebooks_patterns.UnrolledPatternProvider"><code class="flex name class">
|
||
<span>class <span class="ident">UnrolledPatternProvider</span></span>
|
||
<span>(</span><span>n_q: int, flattening: List[int] | None = None, delays: List[int] | None = None)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class UnrolledPatternProvider(CodebooksPatternProvider):
|
||
"""Provider for unrolling codebooks pattern.
|
||
This pattern provider enables to represent the codebook flattened completely or only to some extend
|
||
while also specifying a given delay between the flattened codebooks representation, allowing to
|
||
unroll the codebooks in the sequence.
|
||
|
||
Example:
|
||
1. Flattening of the codebooks.
|
||
By default, the pattern provider will fully flatten the codebooks such as flattening=range(n_q),
|
||
taking n_q = 3 and timesteps = 4:
|
||
[[1, 2, 3, 4],
|
||
[1, 2, 3, 4],
|
||
[1, 2, 3, 4]]
|
||
will result into:
|
||
[[S, S, 1, S, S, 2, S, S, 3, S, S, 4],
|
||
[S, 1, S, S, 2, S, S, 3, S, S, 4, S],
|
||
[1, S, S, 2, S, S, 3, S, S, 4, S, S]]
|
||
2. Partial flattening of the codebooks. The ``flattening`` parameter allows to specify the inner step
|
||
for each of the codebook, allowing to define which codebook to flatten (or keep in parallel), for example
|
||
taking n_q = 3, timesteps = 4 and flattening = [0, 1, 1]:
|
||
[[1, 2, 3, 4],
|
||
[1, 2, 3, 4],
|
||
[1, 2, 3, 4]]
|
||
will result into:
|
||
[[S, 1, S, S, 2, S, S, 3, S, S, 4, S],
|
||
[S, 1, S, S, 2, S, S, 3, S, S, 4, S],
|
||
[1, S, S, 2, S, S, 3, S, S, 4, S, S]]
|
||
3. Flattening with delay. The ``delay`` parameter allows to further unroll the sequence of codebooks
|
||
allowing to specify the delay per codebook. Note that the delay between codebooks flattened to the
|
||
same inner timestep should be coherent. For example, taking n_q = 3, timesteps = 4, flattening = [0, 1, 1]
|
||
and delays = [0, 3, 3]:
|
||
[[1, 2, 3, 4],
|
||
[1, 2, 3, 4],
|
||
[1, 2, 3, 4]]
|
||
will result into:
|
||
[[S, S, S, 1, S, 2, S, 3, S, 4],
|
||
[S, S, S, 1, S, 2, S, 3, S, 4],
|
||
[1, 2, 3, S, 4, S, 5, S, 6, S]]
|
||
|
||
Args:
|
||
n_q (int): Number of codebooks.
|
||
flattening (list of int, optional): Flattening schema over the codebooks. If not defined,
|
||
the codebooks will be flattened to 1 codebook per step, meaning that the sequence will
|
||
have n_q extra steps for each timestep.
|
||
delays (list of int, optional): Delay for each of the codebooks. If not defined,
|
||
no delay is added and therefore will default to [0] * ``n_q``.
|
||
Note that two codebooks that will be flattened to the same inner step
|
||
should have the same delay, otherwise the pattern is considered as invalid.
|
||
"""
|
||
FlattenedCodebook = namedtuple('FlattenedCodebook', ['codebooks', 'delay'])
|
||
|
||
def __init__(self, n_q: int, flattening: tp.Optional[tp.List[int]] = None,
|
||
delays: tp.Optional[tp.List[int]] = None):
|
||
super().__init__(n_q)
|
||
if flattening is None:
|
||
flattening = list(range(n_q))
|
||
if delays is None:
|
||
delays = [0] * n_q
|
||
assert len(flattening) == n_q
|
||
assert len(delays) == n_q
|
||
assert sorted(flattening) == flattening
|
||
assert sorted(delays) == delays
|
||
self._flattened_codebooks = self._build_flattened_codebooks(delays, flattening)
|
||
self.max_delay = max(delays)
|
||
|
||
def _build_flattened_codebooks(self, delays: tp.List[int], flattening: tp.List[int]):
|
||
"""Build a flattened codebooks representation as a dictionary of inner step
|
||
and the actual codebook indices corresponding to the flattened codebook. For convenience, we
|
||
also store the delay associated to the flattened codebook to avoid maintaining an extra mapping.
|
||
"""
|
||
flattened_codebooks: dict = {}
|
||
for q, (inner_step, delay) in enumerate(zip(flattening, delays)):
|
||
if inner_step not in flattened_codebooks:
|
||
flat_codebook = UnrolledPatternProvider.FlattenedCodebook(codebooks=[q], delay=delay)
|
||
else:
|
||
flat_codebook = flattened_codebooks[inner_step]
|
||
assert flat_codebook.delay == delay, (
|
||
"Delay and flattening between codebooks is inconsistent: ",
|
||
"two codebooks flattened to the same position should have the same delay."
|
||
)
|
||
flat_codebook.codebooks.append(q)
|
||
flattened_codebooks[inner_step] = flat_codebook
|
||
return flattened_codebooks
|
||
|
||
@property
|
||
def _num_inner_steps(self):
|
||
"""Number of inner steps to unroll between timesteps in order to flatten the codebooks.
|
||
"""
|
||
return max([inner_step for inner_step in self._flattened_codebooks.keys()]) + 1
|
||
|
||
def num_virtual_steps(self, timesteps: int) -> int:
|
||
return timesteps * self._num_inner_steps + 1
|
||
|
||
def get_pattern(self, timesteps: int) -> Pattern:
|
||
"""Builds pattern for delay across codebooks.
|
||
|
||
Args:
|
||
timesteps (int): Total number of timesteps.
|
||
"""
|
||
# the PatternLayout is built as a tuple of sequence position and list of coordinates
|
||
# so that it can be reordered properly given the required delay between codebooks of given timesteps
|
||
indexed_out: list = [(-1, [])]
|
||
max_timesteps = timesteps + self.max_delay
|
||
for t in range(max_timesteps):
|
||
# for each timestep, we unroll the flattened codebooks,
|
||
# emitting the sequence step with the corresponding delay
|
||
for step in range(self._num_inner_steps):
|
||
if step in self._flattened_codebooks:
|
||
# we have codebooks at this virtual step to emit
|
||
step_codebooks = self._flattened_codebooks[step]
|
||
t_for_q = t + step_codebooks.delay
|
||
coords = [LayoutCoord(t, q) for q in step_codebooks.codebooks]
|
||
if t_for_q < max_timesteps and t < max_timesteps:
|
||
indexed_out.append((t_for_q, coords))
|
||
else:
|
||
# there is no codebook in this virtual step so we emit an empty list
|
||
indexed_out.append((t, []))
|
||
out = [coords for _, coords in sorted(indexed_out)]
|
||
return Pattern(out, n_q=self.n_q, timesteps=timesteps)</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Provider for unrolling codebooks pattern.
|
||
This pattern provider enables to represent the codebook flattened completely or only to some extend
|
||
while also specifying a given delay between the flattened codebooks representation, allowing to
|
||
unroll the codebooks in the sequence.</p>
|
||
<h2 id="example">Example</h2>
|
||
<ol>
|
||
<li>Flattening of the codebooks.
|
||
By default, the pattern provider will fully flatten the codebooks such as flattening=range(n_q),
|
||
taking n_q = 3 and timesteps = 4:
|
||
[[1, 2, 3, 4],
|
||
[1, 2, 3, 4],
|
||
[1, 2, 3, 4]]
|
||
will result into:
|
||
[[S, S, 1, S, S, 2, S, S, 3, S, S, 4],
|
||
[S, 1, S, S, 2, S, S, 3, S, S, 4, S],
|
||
[1, S, S, 2, S, S, 3, S, S, 4, S, S]]</li>
|
||
<li>Partial flattening of the codebooks. The <code>flattening</code> parameter allows to specify the inner step
|
||
for each of the codebook, allowing to define which codebook to flatten (or keep in parallel), for example
|
||
taking n_q = 3, timesteps = 4 and flattening = [0, 1, 1]:
|
||
[[1, 2, 3, 4],
|
||
[1, 2, 3, 4],
|
||
[1, 2, 3, 4]]
|
||
will result into:
|
||
[[S, 1, S, S, 2, S, S, 3, S, S, 4, S],
|
||
[S, 1, S, S, 2, S, S, 3, S, S, 4, S],
|
||
[1, S, S, 2, S, S, 3, S, S, 4, S, S]]</li>
|
||
<li>Flattening with delay. The <code>delay</code> parameter allows to further unroll the sequence of codebooks
|
||
allowing to specify the delay per codebook. Note that the delay between codebooks flattened to the
|
||
same inner timestep should be coherent. For example, taking n_q = 3, timesteps = 4, flattening = [0, 1, 1]
|
||
and delays = [0, 3, 3]:
|
||
[[1, 2, 3, 4],
|
||
[1, 2, 3, 4],
|
||
[1, 2, 3, 4]]
|
||
will result into:
|
||
[[S, S, S, 1, S, 2, S, 3, S, 4],
|
||
[S, S, S, 1, S, 2, S, 3, S, 4],
|
||
[1, 2, 3, S, 4, S, 5, S, 6, S]]</li>
|
||
</ol>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>n_q</code></strong> : <code>int</code></dt>
|
||
<dd>Number of codebooks.</dd>
|
||
<dt><strong><code>flattening</code></strong> : <code>list</code> of <code>int</code>, optional</dt>
|
||
<dd>Flattening schema over the codebooks. If not defined,
|
||
the codebooks will be flattened to 1 codebook per step, meaning that the sequence will
|
||
have n_q extra steps for each timestep.</dd>
|
||
<dt><strong><code>delays</code></strong> : <code>list</code> of <code>int</code>, optional</dt>
|
||
<dd>Delay for each of the codebooks. If not defined,
|
||
no delay is added and therefore will default to [0] * <code>n_q</code>.
|
||
Note that two codebooks that will be flattened to the same inner step
|
||
should have the same delay, otherwise the pattern is considered as invalid.</dd>
|
||
</dl></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.codebooks_patterns.CodebooksPatternProvider" href="#audiocraft.modules.codebooks_patterns.CodebooksPatternProvider">CodebooksPatternProvider</a></li>
|
||
<li>abc.ABC</li>
|
||
</ul>
|
||
<h3>Class variables</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.codebooks_patterns.UnrolledPatternProvider.FlattenedCodebook"><code class="name">var <span class="ident">FlattenedCodebook</span></code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
<h3>Methods</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.codebooks_patterns.UnrolledPatternProvider.get_pattern"><code class="name flex">
|
||
<span>def <span class="ident">get_pattern</span></span>(<span>self, timesteps: int) ‑> <a title="audiocraft.modules.codebooks_patterns.Pattern" href="#audiocraft.modules.codebooks_patterns.Pattern">Pattern</a></span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def get_pattern(self, timesteps: int) -> Pattern:
|
||
"""Builds pattern for delay across codebooks.
|
||
|
||
Args:
|
||
timesteps (int): Total number of timesteps.
|
||
"""
|
||
# the PatternLayout is built as a tuple of sequence position and list of coordinates
|
||
# so that it can be reordered properly given the required delay between codebooks of given timesteps
|
||
indexed_out: list = [(-1, [])]
|
||
max_timesteps = timesteps + self.max_delay
|
||
for t in range(max_timesteps):
|
||
# for each timestep, we unroll the flattened codebooks,
|
||
# emitting the sequence step with the corresponding delay
|
||
for step in range(self._num_inner_steps):
|
||
if step in self._flattened_codebooks:
|
||
# we have codebooks at this virtual step to emit
|
||
step_codebooks = self._flattened_codebooks[step]
|
||
t_for_q = t + step_codebooks.delay
|
||
coords = [LayoutCoord(t, q) for q in step_codebooks.codebooks]
|
||
if t_for_q < max_timesteps and t < max_timesteps:
|
||
indexed_out.append((t_for_q, coords))
|
||
else:
|
||
# there is no codebook in this virtual step so we emit an empty list
|
||
indexed_out.append((t, []))
|
||
out = [coords for _, coords in sorted(indexed_out)]
|
||
return Pattern(out, n_q=self.n_q, timesteps=timesteps)</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Builds pattern for delay across codebooks.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>timesteps</code></strong> : <code>int</code></dt>
|
||
<dd>Total number of timesteps.</dd>
|
||
</dl></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.codebooks_patterns.UnrolledPatternProvider.num_virtual_steps"><code class="name flex">
|
||
<span>def <span class="ident">num_virtual_steps</span></span>(<span>self, timesteps: int) ‑> int</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def num_virtual_steps(self, timesteps: int) -> int:
|
||
return timesteps * self._num_inner_steps + 1</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
</dd>
|
||
</dl>
|
||
</section>
|
||
</article>
|
||
<nav id="sidebar">
|
||
<div class="toc">
|
||
<ul></ul>
|
||
</div>
|
||
<ul id="index">
|
||
<li><h3>Super-module</h3>
|
||
<ul>
|
||
<li><code><a title="audiocraft.modules" href="index.html">audiocraft.modules</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li><h3><a href="#header-classes">Classes</a></h3>
|
||
<ul>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.codebooks_patterns.CoarseFirstPattern" href="#audiocraft.modules.codebooks_patterns.CoarseFirstPattern">CoarseFirstPattern</a></code></h4>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.codebooks_patterns.CodebooksPatternProvider" href="#audiocraft.modules.codebooks_patterns.CodebooksPatternProvider">CodebooksPatternProvider</a></code></h4>
|
||
<ul class="">
|
||
<li><code><a title="audiocraft.modules.codebooks_patterns.CodebooksPatternProvider.get_pattern" href="#audiocraft.modules.codebooks_patterns.CodebooksPatternProvider.get_pattern">get_pattern</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.codebooks_patterns.DelayedPatternProvider" href="#audiocraft.modules.codebooks_patterns.DelayedPatternProvider">DelayedPatternProvider</a></code></h4>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.codebooks_patterns.LayoutCoord" href="#audiocraft.modules.codebooks_patterns.LayoutCoord">LayoutCoord</a></code></h4>
|
||
<ul class="">
|
||
<li><code><a title="audiocraft.modules.codebooks_patterns.LayoutCoord.q" href="#audiocraft.modules.codebooks_patterns.LayoutCoord.q">q</a></code></li>
|
||
<li><code><a title="audiocraft.modules.codebooks_patterns.LayoutCoord.t" href="#audiocraft.modules.codebooks_patterns.LayoutCoord.t">t</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.codebooks_patterns.MusicLMPattern" href="#audiocraft.modules.codebooks_patterns.MusicLMPattern">MusicLMPattern</a></code></h4>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.codebooks_patterns.ParallelPatternProvider" href="#audiocraft.modules.codebooks_patterns.ParallelPatternProvider">ParallelPatternProvider</a></code></h4>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.codebooks_patterns.Pattern" href="#audiocraft.modules.codebooks_patterns.Pattern">Pattern</a></code></h4>
|
||
<ul class="">
|
||
<li><code><a title="audiocraft.modules.codebooks_patterns.Pattern.build_pattern_sequence" href="#audiocraft.modules.codebooks_patterns.Pattern.build_pattern_sequence">build_pattern_sequence</a></code></li>
|
||
<li><code><a title="audiocraft.modules.codebooks_patterns.Pattern.get_first_step_with_timesteps" href="#audiocraft.modules.codebooks_patterns.Pattern.get_first_step_with_timesteps">get_first_step_with_timesteps</a></code></li>
|
||
<li><code><a title="audiocraft.modules.codebooks_patterns.Pattern.get_sequence_coords_with_timestep" href="#audiocraft.modules.codebooks_patterns.Pattern.get_sequence_coords_with_timestep">get_sequence_coords_with_timestep</a></code></li>
|
||
<li><code><a title="audiocraft.modules.codebooks_patterns.Pattern.get_steps_with_timestep" href="#audiocraft.modules.codebooks_patterns.Pattern.get_steps_with_timestep">get_steps_with_timestep</a></code></li>
|
||
<li><code><a title="audiocraft.modules.codebooks_patterns.Pattern.layout" href="#audiocraft.modules.codebooks_patterns.Pattern.layout">layout</a></code></li>
|
||
<li><code><a title="audiocraft.modules.codebooks_patterns.Pattern.max_delay" href="#audiocraft.modules.codebooks_patterns.Pattern.max_delay">max_delay</a></code></li>
|
||
<li><code><a title="audiocraft.modules.codebooks_patterns.Pattern.n_q" href="#audiocraft.modules.codebooks_patterns.Pattern.n_q">n_q</a></code></li>
|
||
<li><code><a title="audiocraft.modules.codebooks_patterns.Pattern.num_sequence_steps" href="#audiocraft.modules.codebooks_patterns.Pattern.num_sequence_steps">num_sequence_steps</a></code></li>
|
||
<li><code><a title="audiocraft.modules.codebooks_patterns.Pattern.revert_pattern_logits" href="#audiocraft.modules.codebooks_patterns.Pattern.revert_pattern_logits">revert_pattern_logits</a></code></li>
|
||
<li><code><a title="audiocraft.modules.codebooks_patterns.Pattern.revert_pattern_sequence" href="#audiocraft.modules.codebooks_patterns.Pattern.revert_pattern_sequence">revert_pattern_sequence</a></code></li>
|
||
<li><code><a title="audiocraft.modules.codebooks_patterns.Pattern.starts_with_special_token" href="#audiocraft.modules.codebooks_patterns.Pattern.starts_with_special_token">starts_with_special_token</a></code></li>
|
||
<li><code><a title="audiocraft.modules.codebooks_patterns.Pattern.timesteps" href="#audiocraft.modules.codebooks_patterns.Pattern.timesteps">timesteps</a></code></li>
|
||
<li><code><a title="audiocraft.modules.codebooks_patterns.Pattern.valid_layout" href="#audiocraft.modules.codebooks_patterns.Pattern.valid_layout">valid_layout</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.codebooks_patterns.UnrolledPatternProvider" href="#audiocraft.modules.codebooks_patterns.UnrolledPatternProvider">UnrolledPatternProvider</a></code></h4>
|
||
<ul class="">
|
||
<li><code><a title="audiocraft.modules.codebooks_patterns.UnrolledPatternProvider.FlattenedCodebook" href="#audiocraft.modules.codebooks_patterns.UnrolledPatternProvider.FlattenedCodebook">FlattenedCodebook</a></code></li>
|
||
<li><code><a title="audiocraft.modules.codebooks_patterns.UnrolledPatternProvider.get_pattern" href="#audiocraft.modules.codebooks_patterns.UnrolledPatternProvider.get_pattern">get_pattern</a></code></li>
|
||
<li><code><a title="audiocraft.modules.codebooks_patterns.UnrolledPatternProvider.num_virtual_steps" href="#audiocraft.modules.codebooks_patterns.UnrolledPatternProvider.num_virtual_steps">num_virtual_steps</a></code></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</nav>
|
||
</main>
|
||
<footer id="footer">
|
||
<p>Generated by <a href="https://pdoc3.github.io/pdoc" title="pdoc: Python API documentation generator"><cite>pdoc</cite> 0.11.5</a>.</p>
|
||
</footer>
|
||
</body>
|
||
</html>
|