facebookresearch--audiocraft
Alexandre Défossez (autodoc)
4126 行
214 KiB
HTML
4126 行
214 KiB
HTML
<!doctype html>
|
||
<html lang="en">
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1, minimum-scale=1">
|
||
<meta name="generator" content="pdoc3 0.11.5">
|
||
<title>audiocraft.modules.conditioners API documentation</title>
|
||
<meta name="description" content="">
|
||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/10up-sanitize.css/13.0.0/sanitize.min.css" integrity="sha512-y1dtMcuvtTMJc1yPgEqF0ZjQbhnc/bFhyvIyVNb9Zk5mIGtqVaAB1Ttl28su8AvFMOY0EwRbAe+HCLqj6W7/KA==" crossorigin>
|
||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/10up-sanitize.css/13.0.0/typography.min.css" integrity="sha512-Y1DYSb995BAfxobCkKepB1BqJJTPrOp3zPL74AWFugHHmmdcvO+C48WLrUOlhGMc0QG7AE3f7gmvvcrmX2fDoA==" crossorigin>
|
||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/default.min.css" crossorigin>
|
||
<style>:root{--highlight-color:#fe9}.flex{display:flex !important}body{line-height:1.5em}#content{padding:20px}#sidebar{padding:1.5em;overflow:hidden}#sidebar > *:last-child{margin-bottom:2cm}.http-server-breadcrumbs{font-size:130%;margin:0 0 15px 0}#footer{font-size:.75em;padding:5px 30px;border-top:1px solid #ddd;text-align:right}#footer p{margin:0 0 0 1em;display:inline-block}#footer p:last-child{margin-right:30px}h1,h2,h3,h4,h5{font-weight:300}h1{font-size:2.5em;line-height:1.1em}h2{font-size:1.75em;margin:2em 0 .50em 0}h3{font-size:1.4em;margin:1.6em 0 .7em 0}h4{margin:0;font-size:105%}h1:target,h2:target,h3:target,h4:target,h5:target,h6:target{background:var(--highlight-color);padding:.2em 0}a{color:#058;text-decoration:none;transition:color .2s ease-in-out}a:visited{color:#503}a:hover{color:#b62}.title code{font-weight:bold}h2[id^="header-"]{margin-top:2em}.ident{color:#900;font-weight:bold}pre code{font-size:.8em;line-height:1.4em;padding:1em;display:block}code{background:#f3f3f3;font-family:"DejaVu Sans Mono",monospace;padding:1px 4px;overflow-wrap:break-word}h1 code{background:transparent}pre{border-top:1px solid #ccc;border-bottom:1px solid #ccc;margin:1em 0}#http-server-module-list{display:flex;flex-flow:column}#http-server-module-list div{display:flex}#http-server-module-list dt{min-width:10%}#http-server-module-list p{margin-top:0}.toc ul,#index{list-style-type:none;margin:0;padding:0}#index code{background:transparent}#index h3{border-bottom:1px solid #ddd}#index ul{padding:0}#index h4{margin-top:.6em;font-weight:bold}@media (min-width:200ex){#index .two-column{column-count:2}}@media (min-width:300ex){#index .two-column{column-count:3}}dl{margin-bottom:2em}dl dl:last-child{margin-bottom:4em}dd{margin:0 0 1em 3em}#header-classes + dl > dd{margin-bottom:3em}dd dd{margin-left:2em}dd p{margin:10px 0}.name{background:#eee;font-size:.85em;padding:5px 10px;display:inline-block;min-width:40%}.name:hover{background:#e0e0e0}dt:target .name{background:var(--highlight-color)}.name > span:first-child{white-space:nowrap}.name.class > span:nth-child(2){margin-left:.4em}.inherited{color:#999;border-left:5px solid #eee;padding-left:1em}.inheritance em{font-style:normal;font-weight:bold}.desc h2{font-weight:400;font-size:1.25em}.desc h3{font-size:1em}.desc dt code{background:inherit}.source > summary,.git-link-div{color:#666;text-align:right;font-weight:400;font-size:.8em;text-transform:uppercase}.source summary > *{white-space:nowrap;cursor:pointer}.git-link{color:inherit;margin-left:1em}.source pre{max-height:500px;overflow:auto;margin:0}.source pre code{font-size:12px;overflow:visible;min-width:max-content}.hlist{list-style:none}.hlist li{display:inline}.hlist li:after{content:',\2002'}.hlist li:last-child:after{content:none}.hlist .hlist{display:inline;padding-left:1em}img{max-width:100%}td{padding:0 .5em}.admonition{padding:.1em 1em;margin:1em 0}.admonition-title{font-weight:bold}.admonition.note,.admonition.info,.admonition.important{background:#aef}.admonition.todo,.admonition.versionadded,.admonition.tip,.admonition.hint{background:#dfd}.admonition.warning,.admonition.versionchanged,.admonition.deprecated{background:#fd4}.admonition.error,.admonition.danger,.admonition.caution{background:lightpink}</style>
|
||
<style media="screen and (min-width: 700px)">@media screen and (min-width:700px){#sidebar{width:30%;height:100vh;overflow:auto;position:sticky;top:0}#content{width:70%;max-width:100ch;padding:3em 4em;border-left:1px solid #ddd}pre code{font-size:1em}.name{font-size:1em}main{display:flex;flex-direction:row-reverse;justify-content:flex-end}.toc ul ul,#index ul ul{padding-left:1em}.toc > ul > li{margin-top:.5em}}</style>
|
||
<style media="print">@media print{#sidebar h1{page-break-before:always}.source{display:none}}@media print{*{background:transparent !important;color:#000 !important;box-shadow:none !important;text-shadow:none !important}a[href]:after{content:" (" attr(href) ")";font-size:90%}a[href][title]:after{content:none}abbr[title]:after{content:" (" attr(title) ")"}.ir a:after,a[href^="javascript:"]:after,a[href^="#"]:after{content:""}pre,blockquote{border:1px solid #999;page-break-inside:avoid}thead{display:table-header-group}tr,img{page-break-inside:avoid}img{max-width:100% !important}@page{margin:0.5cm}p,h2,h3{orphans:3;widows:3}h1,h2,h3,h4,h5,h6{page-break-after:avoid}}</style>
|
||
<script defer src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js" integrity="sha512-D9gUyxqja7hBtkWpPWGt9wfbfaMGVt9gnyCvYa+jojwwPHLCzUm5i8rpk7vD7wNee9bA35eYIjobYPaQuKS1MQ==" crossorigin></script>
|
||
<script>window.addEventListener('DOMContentLoaded', () => {
|
||
hljs.configure({languages: ['bash', 'css', 'diff', 'graphql', 'ini', 'javascript', 'json', 'plaintext', 'python', 'python-repl', 'rust', 'shell', 'sql', 'typescript', 'xml', 'yaml']});
|
||
hljs.highlightAll();
|
||
/* Collapse source docstrings */
|
||
setTimeout(() => {
|
||
[...document.querySelectorAll('.hljs.language-python > .hljs-string')]
|
||
.filter(el => el.innerHTML.length > 200 && ['"""', "'''"].includes(el.innerHTML.substring(0, 3)))
|
||
.forEach(el => {
|
||
let d = document.createElement('details');
|
||
d.classList.add('hljs-string');
|
||
d.innerHTML = '<summary>"""</summary>' + el.innerHTML.substring(3);
|
||
el.replaceWith(d);
|
||
});
|
||
}, 100);
|
||
})</script>
|
||
</head>
|
||
<body>
|
||
<main>
|
||
<article id="content">
|
||
<header>
|
||
<h1 class="title">Module <code>audiocraft.modules.conditioners</code></h1>
|
||
</header>
|
||
<section id="section-intro">
|
||
</section>
|
||
<section>
|
||
</section>
|
||
<section>
|
||
</section>
|
||
<section>
|
||
<h2 class="section-title" id="header-functions">Functions</h2>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.dropout_condition"><code class="name flex">
|
||
<span>def <span class="ident">dropout_condition</span></span>(<span>sample: <a title="audiocraft.modules.conditioners.ConditioningAttributes" href="#audiocraft.modules.conditioners.ConditioningAttributes">ConditioningAttributes</a>,<br>condition_type: str,<br>condition: str,<br>**kwargs) ‑> <a title="audiocraft.modules.conditioners.ConditioningAttributes" href="#audiocraft.modules.conditioners.ConditioningAttributes">ConditioningAttributes</a></span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def dropout_condition(sample: ConditioningAttributes,
|
||
condition_type: str, condition: str,
|
||
**kwargs) -> ConditioningAttributes:
|
||
"""Utility function for nullifying an attribute inside an ConditioningAttributes object.
|
||
If the condition is of type "wav", then nullify it using `nullify_condition` function.
|
||
If the condition is of any other type, set its value to None.
|
||
Works in-place.
|
||
"""
|
||
if condition_type not in ['text', 'wav', 'joint_embed', 'symbolic']:
|
||
raise ValueError(
|
||
"dropout_condition got an unexpected condition type!"
|
||
f" expected 'text', 'wav' or 'joint_embed' but got '{condition_type}'"
|
||
)
|
||
|
||
if condition not in getattr(sample, condition_type):
|
||
raise ValueError(
|
||
"dropout_condition received an unexpected condition!"
|
||
f" expected wav={sample.wav.keys()} and text={sample.text.keys()}"
|
||
f" but got '{condition}' of type '{condition_type}'!"
|
||
)
|
||
|
||
if condition_type == 'wav':
|
||
wav_cond = sample.wav[condition]
|
||
sample.wav[condition] = nullify_wav(wav_cond)
|
||
elif condition_type == 'joint_embed':
|
||
embed = sample.joint_embed[condition]
|
||
sample.joint_embed[condition] = nullify_joint_embed(embed)
|
||
elif condition_type == 'symbolic':
|
||
sample = dropout_symbolic_conditions(sample=sample, condition=condition, **kwargs)
|
||
else:
|
||
sample.text[condition] = None
|
||
|
||
return sample</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Utility function for nullifying an attribute inside an ConditioningAttributes object.
|
||
If the condition is of type "wav", then nullify it using <code><a title="audiocraft.modules.conditioners.nullify_condition" href="#audiocraft.modules.conditioners.nullify_condition">nullify_condition()</a></code> function.
|
||
If the condition is of any other type, set its value to None.
|
||
Works in-place.</p></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.dropout_symbolic_conditions"><code class="name flex">
|
||
<span>def <span class="ident">dropout_symbolic_conditions</span></span>(<span>sample: <a title="audiocraft.modules.conditioners.ConditioningAttributes" href="#audiocraft.modules.conditioners.ConditioningAttributes">ConditioningAttributes</a>,<br>condition: str,<br>null_chord_idx: int = 194) ‑> <a title="audiocraft.modules.conditioners.ConditioningAttributes" href="#audiocraft.modules.conditioners.ConditioningAttributes">ConditioningAttributes</a></span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def dropout_symbolic_conditions(sample: ConditioningAttributes,
|
||
condition: str, null_chord_idx: int = 194) -> ConditioningAttributes:
|
||
"""
|
||
Applies dropout to symbolic conditions within the sample based on the specified condition by setting the condition
|
||
value to a null index.
|
||
Args:
|
||
sample (ConditioningAttributes): The sample containing symbolic attributes to potentially dropout.
|
||
condition (str): The specific condition within the symbolic attributes to apply dropout.
|
||
null_chord_idx (int, optional): The index used to represent a null chord. Defaults to 194.
|
||
Returns:
|
||
ConditioningAttributes: The modified sample with dropout applied to the specified condition.
|
||
Raises:
|
||
ValueError: If the specified condition is not present in the sample's symbolic attributes.
|
||
"""
|
||
if sample.symbolic == {} or sample.symbolic[JascoCondConst.CRD.value].frame_chords.shape[-1] <= 1: # type: ignore
|
||
# nothing to drop
|
||
return sample
|
||
|
||
if condition not in getattr(sample, 'symbolic'):
|
||
raise ValueError(
|
||
"dropout_symbolic_condition received an unexpected condition!"
|
||
f" expected {sample.symbolic.keys()}"
|
||
f" but got '{condition}'!"
|
||
)
|
||
|
||
if condition == JascoCondConst.CRD.value:
|
||
sample.symbolic[condition] = nullify_chords(sample.symbolic[condition], null_chord_idx=null_chord_idx)
|
||
elif condition == JascoCondConst.MLD.value:
|
||
sample.symbolic[condition] = nullify_melody(sample.symbolic[condition])
|
||
|
||
return sample</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Applies dropout to symbolic conditions within the sample based on the specified condition by setting the condition
|
||
value to a null index.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>sample</code></strong> : <code><a title="audiocraft.modules.conditioners.ConditioningAttributes" href="#audiocraft.modules.conditioners.ConditioningAttributes">ConditioningAttributes</a></code></dt>
|
||
<dd>The sample containing symbolic attributes to potentially dropout.</dd>
|
||
<dt><strong><code>condition</code></strong> : <code>str</code></dt>
|
||
<dd>The specific condition within the symbolic attributes to apply dropout.</dd>
|
||
<dt><strong><code>null_chord_idx</code></strong> : <code>int</code>, optional</dt>
|
||
<dd>The index used to represent a null chord. Defaults to 194.</dd>
|
||
</dl>
|
||
<h2 id="returns">Returns</h2>
|
||
<dl>
|
||
<dt><code><a title="audiocraft.modules.conditioners.ConditioningAttributes" href="#audiocraft.modules.conditioners.ConditioningAttributes">ConditioningAttributes</a></code></dt>
|
||
<dd>The modified sample with dropout applied to the specified condition.</dd>
|
||
</dl>
|
||
<h2 id="raises">Raises</h2>
|
||
<dl>
|
||
<dt><code>ValueError</code></dt>
|
||
<dd>If the specified condition is not present in the sample's symbolic attributes.</dd>
|
||
</dl></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.nullify_chords"><code class="name flex">
|
||
<span>def <span class="ident">nullify_chords</span></span>(<span>sym_cond: <a title="audiocraft.modules.conditioners.SymbolicCondition" href="#audiocraft.modules.conditioners.SymbolicCondition">SymbolicCondition</a>,<br>null_chord_idx: int = 194) ‑> <a title="audiocraft.modules.conditioners.SymbolicCondition" href="#audiocraft.modules.conditioners.SymbolicCondition">SymbolicCondition</a></span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def nullify_chords(sym_cond: SymbolicCondition, null_chord_idx: int = 194) -> SymbolicCondition:
|
||
"""Nullify the symbolic condition by setting all frame chords to a specified null chord index.
|
||
Args:
|
||
sym_cond (SymbolicCondition): The symbolic condition containing frame chords to be nullified.
|
||
null_chord_idx (int, optional): The index to use for nullifying the chords. Defaults to 194 (Chordino).
|
||
Returns:
|
||
SymbolicCondition: A new symbolic condition with all frame chords set to the null chord index.
|
||
"""
|
||
return SymbolicCondition(frame_chords=torch.ones_like(sym_cond.frame_chords) * null_chord_idx) # type: ignore</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Nullify the symbolic condition by setting all frame chords to a specified null chord index.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>sym_cond</code></strong> : <code><a title="audiocraft.modules.conditioners.SymbolicCondition" href="#audiocraft.modules.conditioners.SymbolicCondition">SymbolicCondition</a></code></dt>
|
||
<dd>The symbolic condition containing frame chords to be nullified.</dd>
|
||
<dt><strong><code>null_chord_idx</code></strong> : <code>int</code>, optional</dt>
|
||
<dd>The index to use for nullifying the chords. Defaults to 194 (Chordino).</dd>
|
||
</dl>
|
||
<h2 id="returns">Returns</h2>
|
||
<dl>
|
||
<dt><code><a title="audiocraft.modules.conditioners.SymbolicCondition" href="#audiocraft.modules.conditioners.SymbolicCondition">SymbolicCondition</a></code></dt>
|
||
<dd>A new symbolic condition with all frame chords set to the null chord index.</dd>
|
||
</dl></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.nullify_condition"><code class="name flex">
|
||
<span>def <span class="ident">nullify_condition</span></span>(<span>condition: Tuple[torch.Tensor, torch.Tensor], dim: int = 1)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def nullify_condition(condition: ConditionType, dim: int = 1):
|
||
"""Transform an input condition to a null condition.
|
||
The way it is done by converting it to a single zero vector similarly
|
||
to how it is done inside WhiteSpaceTokenizer and NoopTokenizer.
|
||
|
||
Args:
|
||
condition (ConditionType): A tuple of condition and mask (tuple[torch.Tensor, torch.Tensor])
|
||
dim (int): The dimension that will be truncated (should be the time dimension)
|
||
WARNING!: dim should not be the batch dimension!
|
||
Returns:
|
||
ConditionType: A tuple of null condition and mask
|
||
"""
|
||
assert dim != 0, "dim cannot be the batch dimension!"
|
||
assert isinstance(condition, tuple) and \
|
||
isinstance(condition[0], torch.Tensor) and \
|
||
isinstance(condition[1], torch.Tensor), "'nullify_condition' got an unexpected input type!"
|
||
cond, mask = condition
|
||
B = cond.shape[0]
|
||
last_dim = cond.dim() - 1
|
||
out = cond.transpose(dim, last_dim)
|
||
out = 0. * out[..., :1]
|
||
out = out.transpose(dim, last_dim)
|
||
mask = torch.zeros((B, 1), device=out.device).int()
|
||
assert cond.dim() == out.dim()
|
||
return out, mask</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Transform an input condition to a null condition.
|
||
The way it is done by converting it to a single zero vector similarly
|
||
to how it is done inside WhiteSpaceTokenizer and NoopTokenizer.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>condition</code></strong> : <code>ConditionType</code></dt>
|
||
<dd>A tuple of condition and mask (tuple[torch.Tensor, torch.Tensor])</dd>
|
||
<dt><strong><code>dim</code></strong> : <code>int</code></dt>
|
||
<dd>The dimension that will be truncated (should be the time dimension)</dd>
|
||
</dl>
|
||
<p>WARNING!: dim should not be the batch dimension!</p>
|
||
<h2 id="returns">Returns</h2>
|
||
<dl>
|
||
<dt><code>ConditionType</code></dt>
|
||
<dd>A tuple of null condition and mask</dd>
|
||
</dl></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.nullify_joint_embed"><code class="name flex">
|
||
<span>def <span class="ident">nullify_joint_embed</span></span>(<span>embed: <a title="audiocraft.modules.conditioners.JointEmbedCondition" href="#audiocraft.modules.conditioners.JointEmbedCondition">JointEmbedCondition</a>) ‑> <a title="audiocraft.modules.conditioners.JointEmbedCondition" href="#audiocraft.modules.conditioners.JointEmbedCondition">JointEmbedCondition</a></span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def nullify_joint_embed(embed: JointEmbedCondition) -> JointEmbedCondition:
|
||
"""Nullify the joint embedding condition by replacing it by a null tensor, forcing its length to 0,
|
||
and replacing metadata by dummy attributes.
|
||
|
||
Args:
|
||
cond (JointEmbedCondition): Joint embedding condition with wav and text, wav tensor of shape [B, C, T].
|
||
"""
|
||
null_wav, _ = nullify_condition((embed.wav, torch.zeros_like(embed.wav)), dim=embed.wav.dim() - 1)
|
||
return JointEmbedCondition(
|
||
wav=null_wav, text=[None] * len(embed.text),
|
||
length=torch.LongTensor([0]).to(embed.wav.device),
|
||
sample_rate=embed.sample_rate,
|
||
path=[None] * embed.wav.shape[0],
|
||
seek_time=[0] * embed.wav.shape[0],
|
||
)</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Nullify the joint embedding condition by replacing it by a null tensor, forcing its length to 0,
|
||
and replacing metadata by dummy attributes.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>cond</code></strong> : <code><a title="audiocraft.modules.conditioners.JointEmbedCondition" href="#audiocraft.modules.conditioners.JointEmbedCondition">JointEmbedCondition</a></code></dt>
|
||
<dd>Joint embedding condition with wav and text, wav tensor of shape [B, C, T].</dd>
|
||
</dl></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.nullify_melody"><code class="name flex">
|
||
<span>def <span class="ident">nullify_melody</span></span>(<span>sym_cond: <a title="audiocraft.modules.conditioners.SymbolicCondition" href="#audiocraft.modules.conditioners.SymbolicCondition">SymbolicCondition</a>) ‑> <a title="audiocraft.modules.conditioners.SymbolicCondition" href="#audiocraft.modules.conditioners.SymbolicCondition">SymbolicCondition</a></span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def nullify_melody(sym_cond: SymbolicCondition) -> SymbolicCondition:
|
||
"""Nullify the symbolic condition by replacing the melody matrix with zeros matrix.
|
||
Args:
|
||
sym_cond (SymbolicCondition): The symbolic condition containing frame chords to be nullified.
|
||
null_chord_idx (int, optional): The index to use for nullifying the chords. Defaults to 194 (Chordino).
|
||
Returns:
|
||
SymbolicCondition: A new symbolic condition with all frame chords set to the null chord index.
|
||
"""
|
||
return SymbolicCondition(melody=torch.zeros_like(sym_cond.melody)) # type: ignore</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Nullify the symbolic condition by replacing the melody matrix with zeros matrix.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>sym_cond</code></strong> : <code><a title="audiocraft.modules.conditioners.SymbolicCondition" href="#audiocraft.modules.conditioners.SymbolicCondition">SymbolicCondition</a></code></dt>
|
||
<dd>The symbolic condition containing frame chords to be nullified.</dd>
|
||
<dt><strong><code>null_chord_idx</code></strong> : <code>int</code>, optional</dt>
|
||
<dd>The index to use for nullifying the chords. Defaults to 194 (Chordino).</dd>
|
||
</dl>
|
||
<h2 id="returns">Returns</h2>
|
||
<dl>
|
||
<dt><code><a title="audiocraft.modules.conditioners.SymbolicCondition" href="#audiocraft.modules.conditioners.SymbolicCondition">SymbolicCondition</a></code></dt>
|
||
<dd>A new symbolic condition with all frame chords set to the null chord index.</dd>
|
||
</dl></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.nullify_wav"><code class="name flex">
|
||
<span>def <span class="ident">nullify_wav</span></span>(<span>cond: <a title="audiocraft.modules.conditioners.WavCondition" href="#audiocraft.modules.conditioners.WavCondition">WavCondition</a>) ‑> <a title="audiocraft.modules.conditioners.WavCondition" href="#audiocraft.modules.conditioners.WavCondition">WavCondition</a></span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def nullify_wav(cond: WavCondition) -> WavCondition:
|
||
"""Transform a WavCondition to a nullified WavCondition.
|
||
It replaces the wav by a null tensor, forces its length to 0, and replaces metadata by dummy attributes.
|
||
|
||
Args:
|
||
cond (WavCondition): Wav condition with wav, tensor of shape [B, T].
|
||
Returns:
|
||
WavCondition: Nullified wav condition.
|
||
"""
|
||
null_wav, _ = nullify_condition((cond.wav, torch.zeros_like(cond.wav)), dim=cond.wav.dim() - 1)
|
||
return WavCondition(
|
||
wav=null_wav,
|
||
length=torch.tensor([0] * cond.wav.shape[0], device=cond.wav.device),
|
||
sample_rate=cond.sample_rate,
|
||
path=[None] * cond.wav.shape[0],
|
||
seek_time=[None] * cond.wav.shape[0],
|
||
)</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Transform a WavCondition to a nullified WavCondition.
|
||
It replaces the wav by a null tensor, forces its length to 0, and replaces metadata by dummy attributes.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>cond</code></strong> : <code><a title="audiocraft.modules.conditioners.WavCondition" href="#audiocraft.modules.conditioners.WavCondition">WavCondition</a></code></dt>
|
||
<dd>Wav condition with wav, tensor of shape [B, T].</dd>
|
||
</dl>
|
||
<h2 id="returns">Returns</h2>
|
||
<dl>
|
||
<dt><code><a title="audiocraft.modules.conditioners.WavCondition" href="#audiocraft.modules.conditioners.WavCondition">WavCondition</a></code></dt>
|
||
<dd>Nullified wav condition.</dd>
|
||
</dl></div>
|
||
</dd>
|
||
</dl>
|
||
</section>
|
||
<section>
|
||
<h2 class="section-title" id="header-classes">Classes</h2>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.AttributeDropout"><code class="flex name class">
|
||
<span>class <span class="ident">AttributeDropout</span></span>
|
||
<span>(</span><span>p: Dict[str, Dict[str, float]], active_on_eval: bool = False, seed: int = 1234)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class AttributeDropout(DropoutModule):
|
||
"""Dropout with a given probability per attribute.
|
||
This is different from the behavior of ClassifierFreeGuidanceDropout as this allows for attributes
|
||
to be dropped out separately. For example, "artist" can be dropped while "genre" remains.
|
||
This is in contrast to ClassifierFreeGuidanceDropout where if "artist" is dropped "genre"
|
||
must also be dropped.
|
||
|
||
Args:
|
||
p (tp.Dict[str, float]): A dict mapping between attributes and dropout probability. For example:
|
||
...
|
||
"genre": 0.1,
|
||
"artist": 0.5,
|
||
"wav": 0.25,
|
||
...
|
||
active_on_eval (bool, optional): Whether the dropout is active at eval. Default to False.
|
||
seed (int, optional): Random seed.
|
||
"""
|
||
def __init__(self, p: tp.Dict[str, tp.Dict[str, float]], active_on_eval: bool = False, seed: int = 1234):
|
||
super().__init__(seed=seed)
|
||
self.active_on_eval = active_on_eval
|
||
# construct dict that return the values from p otherwise 0
|
||
self.p = {}
|
||
for condition_type, probs in p.items():
|
||
self.p[condition_type] = defaultdict(lambda: 0, probs)
|
||
|
||
def forward(self, samples: tp.List[ConditioningAttributes]) -> tp.List[ConditioningAttributes]:
|
||
"""
|
||
Args:
|
||
samples (list[ConditioningAttributes]): List of conditions.
|
||
Returns:
|
||
list[ConditioningAttributes]: List of conditions after certain attributes were set to None.
|
||
"""
|
||
if not self.training and not self.active_on_eval:
|
||
return samples
|
||
|
||
samples = deepcopy(samples)
|
||
for condition_type, ps in self.p.items(): # for condition types [text, wav, symbolic]
|
||
for condition, p in ps.items(): # for attributes of each type (e.g., [artist, genre])
|
||
if torch.rand(1, generator=self.rng).item() < p:
|
||
for sample in samples:
|
||
dropout_condition(sample, condition_type, condition)
|
||
return samples
|
||
|
||
def __repr__(self):
|
||
return f"AttributeDropout({dict(self.p)})"</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Dropout with a given probability per attribute.
|
||
This is different from the behavior of ClassifierFreeGuidanceDropout as this allows for attributes
|
||
to be dropped out separately. For example, "artist" can be dropped while "genre" remains.
|
||
This is in contrast to ClassifierFreeGuidanceDropout where if "artist" is dropped "genre"
|
||
must also be dropped.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>p</code></strong> : <code>tp.Dict[str, float]</code></dt>
|
||
<dd>A dict mapping between attributes and dropout probability. For example:
|
||
…
|
||
"genre": 0.1,
|
||
"artist": 0.5,
|
||
"wav": 0.25,
|
||
…</dd>
|
||
<dt><strong><code>active_on_eval</code></strong> : <code>bool</code>, optional</dt>
|
||
<dd>Whether the dropout is active at eval. Default to False.</dd>
|
||
<dt><strong><code>seed</code></strong> : <code>int</code>, optional</dt>
|
||
<dd>Random seed.</dd>
|
||
</dl>
|
||
<p>Initializes internal Module state, shared by both nn.Module and ScriptModule.</p></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.conditioners.DropoutModule" href="#audiocraft.modules.conditioners.DropoutModule">DropoutModule</a></li>
|
||
<li>torch.nn.modules.module.Module</li>
|
||
</ul>
|
||
<h3>Class variables</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.AttributeDropout.call_super_init"><code class="name">var <span class="ident">call_super_init</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.AttributeDropout.dump_patches"><code class="name">var <span class="ident">dump_patches</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.AttributeDropout.training"><code class="name">var <span class="ident">training</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
<h3>Methods</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.AttributeDropout.forward"><code class="name flex">
|
||
<span>def <span class="ident">forward</span></span>(<span>self,<br>samples: List[<a title="audiocraft.modules.conditioners.ConditioningAttributes" href="#audiocraft.modules.conditioners.ConditioningAttributes">ConditioningAttributes</a>]) ‑> List[<a title="audiocraft.modules.conditioners.ConditioningAttributes" href="#audiocraft.modules.conditioners.ConditioningAttributes">ConditioningAttributes</a>]</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def forward(self, samples: tp.List[ConditioningAttributes]) -> tp.List[ConditioningAttributes]:
|
||
"""
|
||
Args:
|
||
samples (list[ConditioningAttributes]): List of conditions.
|
||
Returns:
|
||
list[ConditioningAttributes]: List of conditions after certain attributes were set to None.
|
||
"""
|
||
if not self.training and not self.active_on_eval:
|
||
return samples
|
||
|
||
samples = deepcopy(samples)
|
||
for condition_type, ps in self.p.items(): # for condition types [text, wav, symbolic]
|
||
for condition, p in ps.items(): # for attributes of each type (e.g., [artist, genre])
|
||
if torch.rand(1, generator=self.rng).item() < p:
|
||
for sample in samples:
|
||
dropout_condition(sample, condition_type, condition)
|
||
return samples</code></pre>
|
||
</details>
|
||
<div class="desc"><h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>samples</code></strong> : <code>list[<a title="audiocraft.modules.conditioners.ConditioningAttributes" href="#audiocraft.modules.conditioners.ConditioningAttributes">ConditioningAttributes</a>]</code></dt>
|
||
<dd>List of conditions.</dd>
|
||
</dl>
|
||
<h2 id="returns">Returns</h2>
|
||
<dl>
|
||
<dt><code>list[<a title="audiocraft.modules.conditioners.ConditioningAttributes" href="#audiocraft.modules.conditioners.ConditioningAttributes">ConditioningAttributes</a>]</code></dt>
|
||
<dd>List of conditions after certain attributes were set to None.</dd>
|
||
</dl></div>
|
||
</dd>
|
||
</dl>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.BaseConditioner"><code class="flex name class">
|
||
<span>class <span class="ident">BaseConditioner</span></span>
|
||
<span>(</span><span>dim: int, output_dim: int)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class BaseConditioner(nn.Module):
|
||
"""Base model for all conditioner modules.
|
||
We allow the output dim to be different than the hidden dim for two reasons:
|
||
1) keep our LUTs small when the vocab is large;
|
||
2) make all condition dims consistent.
|
||
|
||
Args:
|
||
dim (int): Hidden dim of the model.
|
||
output_dim (int): Output dim of the conditioner.
|
||
"""
|
||
def __init__(self, dim: int, output_dim: int):
|
||
super().__init__()
|
||
self.dim = dim
|
||
self.output_dim = output_dim
|
||
if self.output_dim > -1: # omit projection when output_dim <= 0
|
||
self.output_proj = nn.Linear(dim, output_dim)
|
||
|
||
def tokenize(self, *args, **kwargs) -> tp.Any:
|
||
"""Should be any part of the processing that will lead to a synchronization
|
||
point, e.g. BPE tokenization with transfer to the GPU.
|
||
|
||
The returned value will be saved and return later when calling forward().
|
||
"""
|
||
raise NotImplementedError()
|
||
|
||
def forward(self, inputs: tp.Any) -> ConditionType:
|
||
"""Gets input that should be used as conditioning (e.g, genre, description or a waveform).
|
||
Outputs a ConditionType, after the input data was embedded as a dense vector.
|
||
|
||
Returns:
|
||
ConditionType:
|
||
- A tensor of size [B, T, D] where B is the batch size, T is the length of the
|
||
output embedding and D is the dimension of the embedding.
|
||
- And a mask indicating where the padding tokens.
|
||
"""
|
||
raise NotImplementedError()</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Base model for all conditioner modules.
|
||
We allow the output dim to be different than the hidden dim for two reasons:
|
||
1) keep our LUTs small when the vocab is large;
|
||
2) make all condition dims consistent.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>dim</code></strong> : <code>int</code></dt>
|
||
<dd>Hidden dim of the model.</dd>
|
||
<dt><strong><code>output_dim</code></strong> : <code>int</code></dt>
|
||
<dd>Output dim of the conditioner.</dd>
|
||
</dl>
|
||
<p>Initializes internal Module state, shared by both nn.Module and ScriptModule.</p></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li>torch.nn.modules.module.Module</li>
|
||
</ul>
|
||
<h3>Subclasses</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.conditioners.JointEmbeddingConditioner" href="#audiocraft.modules.conditioners.JointEmbeddingConditioner">JointEmbeddingConditioner</a></li>
|
||
<li><a title="audiocraft.modules.conditioners.TextConditioner" href="#audiocraft.modules.conditioners.TextConditioner">TextConditioner</a></li>
|
||
<li><a title="audiocraft.modules.conditioners.WaveformConditioner" href="#audiocraft.modules.conditioners.WaveformConditioner">WaveformConditioner</a></li>
|
||
<li><a title="audiocraft.modules.jasco_conditioners.ChordsEmbConditioner" href="jasco_conditioners.html#audiocraft.modules.jasco_conditioners.ChordsEmbConditioner">ChordsEmbConditioner</a></li>
|
||
<li><a title="audiocraft.modules.jasco_conditioners.MelodyConditioner" href="jasco_conditioners.html#audiocraft.modules.jasco_conditioners.MelodyConditioner">MelodyConditioner</a></li>
|
||
</ul>
|
||
<h3>Class variables</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.BaseConditioner.call_super_init"><code class="name">var <span class="ident">call_super_init</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.BaseConditioner.dump_patches"><code class="name">var <span class="ident">dump_patches</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.BaseConditioner.training"><code class="name">var <span class="ident">training</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
<h3>Methods</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.BaseConditioner.forward"><code class="name flex">
|
||
<span>def <span class="ident">forward</span></span>(<span>self, inputs: Any) ‑> Tuple[torch.Tensor, torch.Tensor]</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def forward(self, inputs: tp.Any) -> ConditionType:
|
||
"""Gets input that should be used as conditioning (e.g, genre, description or a waveform).
|
||
Outputs a ConditionType, after the input data was embedded as a dense vector.
|
||
|
||
Returns:
|
||
ConditionType:
|
||
- A tensor of size [B, T, D] where B is the batch size, T is the length of the
|
||
output embedding and D is the dimension of the embedding.
|
||
- And a mask indicating where the padding tokens.
|
||
"""
|
||
raise NotImplementedError()</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Gets input that should be used as conditioning (e.g, genre, description or a waveform).
|
||
Outputs a ConditionType, after the input data was embedded as a dense vector.</p>
|
||
<h2 id="returns">Returns</h2>
|
||
<p>ConditionType:
|
||
- A tensor of size [B, T, D] where B is the batch size, T is the length of the
|
||
output embedding and D is the dimension of the embedding.
|
||
- And a mask indicating where the padding tokens.</p></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.BaseConditioner.tokenize"><code class="name flex">
|
||
<span>def <span class="ident">tokenize</span></span>(<span>self, *args, **kwargs) ‑> Any</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def tokenize(self, *args, **kwargs) -> tp.Any:
|
||
"""Should be any part of the processing that will lead to a synchronization
|
||
point, e.g. BPE tokenization with transfer to the GPU.
|
||
|
||
The returned value will be saved and return later when calling forward().
|
||
"""
|
||
raise NotImplementedError()</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Should be any part of the processing that will lead to a synchronization
|
||
point, e.g. BPE tokenization with transfer to the GPU.</p>
|
||
<p>The returned value will be saved and return later when calling forward().</p></div>
|
||
</dd>
|
||
</dl>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.CLAPEmbeddingConditioner"><code class="flex name class">
|
||
<span>class <span class="ident">CLAPEmbeddingConditioner</span></span>
|
||
<span>(</span><span>dim: int,<br>output_dim: int,<br>device: str,<br>attribute: str,<br>quantize: bool,<br>n_q: int,<br>bins: int,<br>checkpoint: str | pathlib.Path,<br>model_arch: str,<br>enable_fusion: bool,<br>sample_rate: int,<br>max_audio_length: int,<br>audio_stride: int,<br>normalize: bool,<br>text_p: bool,<br>batch_size: int | None = None,<br>autocast_dtype: str | None = 'float32',<br>cache_path: str | None = None,<br>**kwargs)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class CLAPEmbeddingConditioner(JointEmbeddingConditioner):
|
||
"""Joint Embedding conditioner based on pre-trained CLAP model.
|
||
|
||
This CLAP-based conditioner supports a caching mechanism
|
||
over the computed embeddings for faster training.
|
||
|
||
Args:
|
||
dim (int): Dimension.
|
||
output_dim (int): Output dimension.
|
||
device (str): Device.
|
||
attribute (str): Attribute used by the conditioner.
|
||
quantize (bool): Whether to quantize the CLAP embedding.
|
||
n_q (int): Number of residual quantizers (used if quantize is true).
|
||
bins (int): Quantizers' codebooks size (used if quantize is true).
|
||
checkpoint (str): Path to CLAP checkpoint.
|
||
model_arch (str): CLAP model architecture.
|
||
enable_fusion (bool): Enable fusion for CLAP model.
|
||
sample_rate (int): Sample rate used by CLAP model.
|
||
max_audio_length (float): Maximum audio length for CLAP model.
|
||
audio_stride (float): Stride to use for getting a CLAP embedding on the full sequence.
|
||
normalize (bool): Whether to normalize the CLAP embedding.
|
||
text_p (float): Probability of using text representation instead of audio at train time.
|
||
batch_size (Optional[int]): Batch size for CLAP embedding computation.
|
||
autocast_dtype (str): Autocast for the conditioner.
|
||
cache_path (Optional[str]): Path for pre-computed embeddings caching.
|
||
kwargs: Additional parameters for residual vector quantizer.
|
||
"""
|
||
def __init__(self, dim: int, output_dim: int, device: str, attribute: str,
|
||
quantize: bool, n_q: int, bins: int, checkpoint: tp.Union[str, Path], model_arch: str,
|
||
enable_fusion: bool, sample_rate: int, max_audio_length: int, audio_stride: int,
|
||
normalize: bool, text_p: bool, batch_size: tp.Optional[int] = None,
|
||
autocast_dtype: tp.Optional[str] = 'float32', cache_path: tp.Optional[str] = None, **kwargs):
|
||
try:
|
||
import laion_clap # type: ignore
|
||
except ImportError:
|
||
raise ImportError("Please install CLAP to use the CLAPEmbeddingConditioner: 'pip install laion_clap'")
|
||
warnings.warn("Sample rate for CLAP conditioner was fixed in version v1.1.0, (from 44.1 to 48 kHz). "
|
||
"Please retrain all models.")
|
||
checkpoint = AudioCraftEnvironment.resolve_reference_path(checkpoint)
|
||
clap_tokenize = RobertaTokenizer.from_pretrained('roberta-base')
|
||
clap_model = laion_clap.CLAP_Module(enable_fusion=enable_fusion, amodel=model_arch)
|
||
load_clap_state_dict(clap_model, checkpoint)
|
||
clap_model.eval()
|
||
clap_model.to(device)
|
||
super().__init__(dim=dim, output_dim=output_dim, device=device, attribute=attribute,
|
||
autocast_dtype=autocast_dtype, quantize=quantize, n_q=n_q, bins=bins,
|
||
**kwargs)
|
||
self.checkpoint = checkpoint
|
||
self.enable_fusion = enable_fusion
|
||
self.model_arch = model_arch
|
||
self.clap: laion_clap.CLAP_Module
|
||
self.clap_tokenize: RobertaTokenizer
|
||
self.clap_sample_rate = sample_rate
|
||
self.clap_max_frames = int(self.clap_sample_rate * max_audio_length)
|
||
self.clap_stride = int(self.clap_sample_rate * audio_stride)
|
||
self.batch_size = batch_size or 1
|
||
self.normalize = normalize
|
||
self.text_p = text_p
|
||
self.__dict__['clap_tokenize'] = clap_tokenize
|
||
self.__dict__['clap'] = clap_model
|
||
self.wav_cache, self.text_cache = None, None
|
||
if cache_path is not None:
|
||
self.wav_cache = EmbeddingCache(Path(cache_path) / 'wav', self.device,
|
||
compute_embed_fn=self._get_wav_embedding_for_cache,
|
||
extract_embed_fn=self._extract_wav_embedding_chunk)
|
||
self.text_cache = EmbeddingCache(Path(cache_path) / 'text', self.device,
|
||
compute_embed_fn=self._get_text_embedding_for_cache)
|
||
|
||
def _tokenizer(self, texts: tp.Union[str, tp.List[str]]) -> dict:
|
||
# we use the default params from CLAP module here as well
|
||
return self.clap_tokenize(texts, padding="max_length", truncation=True, max_length=77, return_tensors="pt")
|
||
|
||
def _compute_text_embedding(self, text: tp.List[str]) -> torch.Tensor:
|
||
"""Compute text embedding from CLAP model on a given a batch of text.
|
||
|
||
Args:
|
||
text (list[str]): List of text for the batch, with B items.
|
||
Returns:
|
||
torch.Tensor: CLAP embedding derived from text, of shape [B, 1, D], with D the CLAP embedding dimension.
|
||
"""
|
||
with torch.no_grad():
|
||
embed = self.clap.get_text_embedding(text, tokenizer=self._tokenizer, use_tensor=True)
|
||
return embed.view(embed.size(0), 1, embed.size(-1))
|
||
|
||
def _get_text_embedding_for_cache(self, path: tp.Union[Path, str],
|
||
x: JointEmbedCondition, idx: int) -> torch.Tensor:
|
||
"""Get text embedding function for the cache."""
|
||
text = x.text[idx]
|
||
text = text if text is not None else ""
|
||
return self._compute_text_embedding([text])[0]
|
||
|
||
def _preprocess_wav(self, wav: torch.Tensor, length: torch.Tensor, sample_rates: tp.List[int]) -> torch.Tensor:
|
||
"""Preprocess wav to expected format by CLAP model.
|
||
|
||
Args:
|
||
wav (torch.Tensor): Audio wav, of shape [B, C, T].
|
||
length (torch.Tensor): Actual length of the audio for each item in the batch, of shape [B].
|
||
sample_rates (list[int]): Sample rates for each sample in the batch
|
||
Returns:
|
||
torch.Tensor: Audio wav of shape [B, T].
|
||
"""
|
||
assert wav.dim() == 3, "Expecting wav to be [B, C, T]"
|
||
if sample_rates is not None:
|
||
_wav = []
|
||
for i, audio in enumerate(wav):
|
||
sr = sample_rates[i]
|
||
audio = convert_audio(audio, from_rate=sr, to_rate=self.clap_sample_rate, to_channels=1)
|
||
_wav.append(audio)
|
||
wav = torch.stack(_wav, dim=0)
|
||
wav = wav.mean(dim=1)
|
||
return wav
|
||
|
||
def _compute_wav_embedding(self, wav: torch.Tensor, length: torch.Tensor,
|
||
sample_rates: tp.List[int], reduce_mean: bool = False) -> torch.Tensor:
|
||
"""Compute audio wave embedding from CLAP model.
|
||
|
||
Since CLAP operates on a fixed sequence length audio inputs and we need to process longer audio sequences,
|
||
we calculate the wav embeddings on `clap_max_frames` windows with `clap_stride`-second stride and
|
||
average the resulting embeddings.
|
||
|
||
Args:
|
||
wav (torch.Tensor): Audio wav, of shape [B, C, T].
|
||
length (torch.Tensor): Actual length of the audio for each item in the batch, of shape [B].
|
||
sample_rates (list[int]): Sample rates for each sample in the batch.
|
||
reduce_mean (bool): Whether to get the average tensor.
|
||
Returns:
|
||
torch.Tensor: Audio embedding of shape [B, F, D], F being the number of chunks, D the dimension.
|
||
"""
|
||
with torch.no_grad():
|
||
wav = self._preprocess_wav(wav, length, sample_rates)
|
||
B, T = wav.shape
|
||
if T >= self.clap_max_frames:
|
||
wav = wav.unfold(-1, self.clap_max_frames, self.clap_stride) # [B, F, T]
|
||
else:
|
||
wav = wav.view(-1, 1, T) # [B, F, T] with F=1
|
||
wav = einops.rearrange(wav, 'b f t -> (b f) t')
|
||
embed_list = []
|
||
for i in range(0, wav.size(0), self.batch_size):
|
||
_wav = wav[i:i+self.batch_size, ...]
|
||
_embed = self.clap.get_audio_embedding_from_data(_wav, use_tensor=True)
|
||
embed_list.append(_embed)
|
||
embed = torch.cat(embed_list, dim=0)
|
||
embed = einops.rearrange(embed, '(b f) d -> b f d', b=B)
|
||
if reduce_mean:
|
||
embed = embed.mean(dim=1, keepdim=True)
|
||
return embed # [B, F, D] with F=1 if reduce_mean is True
|
||
|
||
def _get_wav_embedding_for_cache(self, path: tp.Union[str, Path],
|
||
x: JointEmbedCondition, idx: int) -> torch.Tensor:
|
||
"""Compute audio wave embedding for the cache.
|
||
The embedding is computed on a given audio read from file.
|
||
|
||
Args:
|
||
path (str or Path): Path to the full audio file.
|
||
Returns:
|
||
torch.Tensor: Single-item tensor of shape [F, D], F being the number of chunks, D the dimension.
|
||
"""
|
||
wav, sr = audio_read(path) # [C, T]
|
||
wav = wav.unsqueeze(0).to(self.device) # [1, C, T]
|
||
wav_len = torch.LongTensor([wav.shape[-1]]).to(self.device)
|
||
embed = self._compute_wav_embedding(wav, wav_len, [sr], reduce_mean=False) # [B, F, D]
|
||
return embed.squeeze(0) # [F, D]
|
||
|
||
def _extract_wav_embedding_chunk(self, full_embed: torch.Tensor, x: JointEmbedCondition, idx: int) -> torch.Tensor:
|
||
"""Extract the chunk of embedding matching the seek_time and length from the full CLAP audio embedding.
|
||
|
||
Args:
|
||
full_embed (torch.Tensor): CLAP embedding computed on the full wave, of shape [F, D].
|
||
x (JointEmbedCondition): Joint embedding condition for the full batch.
|
||
idx (int): Index considered for the given embedding to extract.
|
||
Returns:
|
||
torch.Tensor: Wav embedding averaged on sliding window, of shape [1, D].
|
||
"""
|
||
sample_rate = x.sample_rate[idx]
|
||
seek_time = x.seek_time[idx]
|
||
seek_time = 0. if seek_time is None else seek_time
|
||
clap_stride = int(self.clap_stride / self.clap_sample_rate) * sample_rate
|
||
end_seek_time = seek_time + self.clap_max_frames / self.clap_sample_rate
|
||
start_offset = int(seek_time * sample_rate // clap_stride)
|
||
end_offset = int(end_seek_time * sample_rate // clap_stride)
|
||
wav_embed = full_embed[start_offset:end_offset, ...]
|
||
wav_embed = wav_embed.mean(dim=0, keepdim=True)
|
||
return wav_embed.to(self.device) # [F, D]
|
||
|
||
def _get_text_embedding(self, x: JointEmbedCondition) -> torch.Tensor:
|
||
"""Get CLAP embedding from a batch of text descriptions."""
|
||
no_nullified_cond = x.wav.shape[-1] > 1 # we don't want to read from cache when condition dropout
|
||
if self.text_cache is not None and no_nullified_cond:
|
||
assert all(p is not None for p in x.path), "Cache requires all JointEmbedCondition paths to be provided"
|
||
paths = [Path(p) for p in x.path if p is not None]
|
||
embed = self.text_cache.get_embed_from_cache(paths, x)
|
||
else:
|
||
text = [xi if xi is not None else "" for xi in x.text]
|
||
embed = self._compute_text_embedding(text)
|
||
if self.normalize:
|
||
embed = torch.nn.functional.normalize(embed, p=2.0, dim=-1)
|
||
return embed
|
||
|
||
def _get_wav_embedding(self, x: JointEmbedCondition) -> torch.Tensor:
|
||
"""Get CLAP embedding from a batch of audio tensors (and corresponding sample rates)."""
|
||
no_undefined_paths = all(p is not None for p in x.path)
|
||
no_nullified_cond = x.wav.shape[-1] > 1 # we don't want to read from cache when condition dropout
|
||
if self.wav_cache is not None and no_undefined_paths and no_nullified_cond:
|
||
paths = [Path(p) for p in x.path if p is not None]
|
||
embed = self.wav_cache.get_embed_from_cache(paths, x)
|
||
else:
|
||
embed = self._compute_wav_embedding(x.wav, x.length, x.sample_rate, reduce_mean=True)
|
||
if self.normalize:
|
||
embed = torch.nn.functional.normalize(embed, p=2.0, dim=-1)
|
||
return embed
|
||
|
||
def tokenize(self, x: JointEmbedCondition) -> JointEmbedCondition:
|
||
# Trying to limit as much as possible sync points when the cache is warm.
|
||
no_undefined_paths = all(p is not None for p in x.path)
|
||
if self.wav_cache is not None and no_undefined_paths:
|
||
assert all([p is not None for p in x.path]), "Cache requires all JointEmbedCondition paths to be provided"
|
||
paths = [Path(p) for p in x.path if p is not None]
|
||
self.wav_cache.populate_embed_cache(paths, x)
|
||
if self.text_cache is not None and no_undefined_paths:
|
||
assert all([p is not None for p in x.path]), "Cache requires all JointEmbedCondition paths to be provided"
|
||
paths = [Path(p) for p in x.path if p is not None]
|
||
self.text_cache.populate_embed_cache(paths, x)
|
||
return x
|
||
|
||
def _get_embed(self, x: JointEmbedCondition) -> tp.Tuple[torch.Tensor, torch.Tensor]:
|
||
"""Extract shared latent representation from either the wav or the text using CLAP."""
|
||
# decide whether to use text embedding at train time or not
|
||
use_text_embed = random.random() < self.text_p
|
||
if self.training and not use_text_embed:
|
||
embed = self._get_wav_embedding(x)
|
||
empty_idx = torch.LongTensor([]) # we assume we always have the audio wav
|
||
else:
|
||
embed = self._get_text_embedding(x)
|
||
empty_idx = torch.LongTensor([i for i, xi in enumerate(x.text) if xi is None or xi == ""])
|
||
return embed, empty_idx</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Joint Embedding conditioner based on pre-trained CLAP model.</p>
|
||
<p>This CLAP-based conditioner supports a caching mechanism
|
||
over the computed embeddings for faster training.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>dim</code></strong> : <code>int</code></dt>
|
||
<dd>Dimension.</dd>
|
||
<dt><strong><code>output_dim</code></strong> : <code>int</code></dt>
|
||
<dd>Output dimension.</dd>
|
||
<dt><strong><code>device</code></strong> : <code>str</code></dt>
|
||
<dd>Device.</dd>
|
||
<dt><strong><code>attribute</code></strong> : <code>str</code></dt>
|
||
<dd>Attribute used by the conditioner.</dd>
|
||
<dt><strong><code>quantize</code></strong> : <code>bool</code></dt>
|
||
<dd>Whether to quantize the CLAP embedding.</dd>
|
||
<dt><strong><code>n_q</code></strong> : <code>int</code></dt>
|
||
<dd>Number of residual quantizers (used if quantize is true).</dd>
|
||
<dt><strong><code>bins</code></strong> : <code>int</code></dt>
|
||
<dd>Quantizers' codebooks size (used if quantize is true).</dd>
|
||
<dt><strong><code>checkpoint</code></strong> : <code>str</code></dt>
|
||
<dd>Path to CLAP checkpoint.</dd>
|
||
<dt><strong><code>model_arch</code></strong> : <code>str</code></dt>
|
||
<dd>CLAP model architecture.</dd>
|
||
<dt><strong><code>enable_fusion</code></strong> : <code>bool</code></dt>
|
||
<dd>Enable fusion for CLAP model.</dd>
|
||
<dt><strong><code>sample_rate</code></strong> : <code>int</code></dt>
|
||
<dd>Sample rate used by CLAP model.</dd>
|
||
<dt><strong><code>max_audio_length</code></strong> : <code>float</code></dt>
|
||
<dd>Maximum audio length for CLAP model.</dd>
|
||
<dt><strong><code>audio_stride</code></strong> : <code>float</code></dt>
|
||
<dd>Stride to use for getting a CLAP embedding on the full sequence.</dd>
|
||
<dt><strong><code>normalize</code></strong> : <code>bool</code></dt>
|
||
<dd>Whether to normalize the CLAP embedding.</dd>
|
||
<dt><strong><code>text_p</code></strong> : <code>float</code></dt>
|
||
<dd>Probability of using text representation instead of audio at train time.</dd>
|
||
<dt><strong><code>batch_size</code></strong> : <code>Optional[int]</code></dt>
|
||
<dd>Batch size for CLAP embedding computation.</dd>
|
||
<dt><strong><code>autocast_dtype</code></strong> : <code>str</code></dt>
|
||
<dd>Autocast for the conditioner.</dd>
|
||
<dt><strong><code>cache_path</code></strong> : <code>Optional[str]</code></dt>
|
||
<dd>Path for pre-computed embeddings caching.</dd>
|
||
<dt><strong><code>kwargs</code></strong></dt>
|
||
<dd>Additional parameters for residual vector quantizer.</dd>
|
||
</dl>
|
||
<p>Initializes internal Module state, shared by both nn.Module and ScriptModule.</p></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.conditioners.JointEmbeddingConditioner" href="#audiocraft.modules.conditioners.JointEmbeddingConditioner">JointEmbeddingConditioner</a></li>
|
||
<li><a title="audiocraft.modules.conditioners.BaseConditioner" href="#audiocraft.modules.conditioners.BaseConditioner">BaseConditioner</a></li>
|
||
<li>torch.nn.modules.module.Module</li>
|
||
</ul>
|
||
<h3>Class variables</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.CLAPEmbeddingConditioner.call_super_init"><code class="name">var <span class="ident">call_super_init</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.CLAPEmbeddingConditioner.dump_patches"><code class="name">var <span class="ident">dump_patches</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.CLAPEmbeddingConditioner.training"><code class="name">var <span class="ident">training</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
<h3>Inherited members</h3>
|
||
<ul class="hlist">
|
||
<li><code><b><a title="audiocraft.modules.conditioners.JointEmbeddingConditioner" href="#audiocraft.modules.conditioners.JointEmbeddingConditioner">JointEmbeddingConditioner</a></b></code>:
|
||
<ul class="hlist">
|
||
<li><code><a title="audiocraft.modules.conditioners.JointEmbeddingConditioner.forward" href="#audiocraft.modules.conditioners.BaseConditioner.forward">forward</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.JointEmbeddingConditioner.tokenize" href="#audiocraft.modules.conditioners.BaseConditioner.tokenize">tokenize</a></code></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.ChromaStemConditioner"><code class="flex name class">
|
||
<span>class <span class="ident">ChromaStemConditioner</span></span>
|
||
<span>(</span><span>output_dim: int,<br>sample_rate: int,<br>n_chroma: int,<br>radix2_exp: int,<br>duration: float,<br>match_len_on_eval: bool = True,<br>eval_wavs: str | None = None,<br>n_eval_wavs: int = 0,<br>cache_path: pathlib.Path | str | None = None,<br>device: torch.device | str = 'cpu',<br>**kwargs)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class ChromaStemConditioner(WaveformConditioner):
|
||
"""Chroma conditioner based on stems.
|
||
The ChromaStemConditioner uses DEMUCS to first filter out drums and bass, as
|
||
the drums and bass often dominate the chroma leading to the chroma features
|
||
not containing information about the melody.
|
||
|
||
Args:
|
||
output_dim (int): Output dimension for the conditioner.
|
||
sample_rate (int): Sample rate for the chroma extractor.
|
||
n_chroma (int): Number of chroma bins for the chroma extractor.
|
||
radix2_exp (int): Size of stft window for the chroma extractor (power of 2, e.g. 12 -> 2^12).
|
||
duration (int): duration used during training. This is later used for correct padding
|
||
in case we are using chroma as prefix.
|
||
match_len_on_eval (bool, optional): if True then all chromas are padded to the training
|
||
duration. Defaults to False.
|
||
eval_wavs (str, optional): path to a dataset manifest with waveform, this waveforms are used as
|
||
conditions during eval (for cases where we don't want to leak test conditions like MusicCaps).
|
||
Defaults to None.
|
||
n_eval_wavs (int, optional): limits the number of waveforms used for conditioning. Defaults to 0.
|
||
device (tp.Union[torch.device, str], optional): Device for the conditioner.
|
||
**kwargs: Additional parameters for the chroma extractor.
|
||
"""
|
||
def __init__(self, output_dim: int, sample_rate: int, n_chroma: int, radix2_exp: int,
|
||
duration: float, match_len_on_eval: bool = True, eval_wavs: tp.Optional[str] = None,
|
||
n_eval_wavs: int = 0, cache_path: tp.Optional[tp.Union[str, Path]] = None,
|
||
device: tp.Union[torch.device, str] = 'cpu', **kwargs):
|
||
from demucs import pretrained
|
||
super().__init__(dim=n_chroma, output_dim=output_dim, device=device)
|
||
self.autocast = TorchAutocast(enabled=device != 'cpu', device_type=self.device, dtype=torch.float32)
|
||
self.sample_rate = sample_rate
|
||
self.match_len_on_eval = match_len_on_eval
|
||
if match_len_on_eval:
|
||
self._use_masking = False
|
||
self.duration = duration
|
||
self.__dict__['demucs'] = pretrained.get_model('htdemucs').to(device)
|
||
stem_sources: list = self.demucs.sources # type: ignore
|
||
self.stem_indices = torch.LongTensor([stem_sources.index('vocals'), stem_sources.index('other')]).to(device)
|
||
self.chroma = ChromaExtractor(sample_rate=sample_rate, n_chroma=n_chroma,
|
||
radix2_exp=radix2_exp, **kwargs).to(device)
|
||
self.chroma_len = self._get_chroma_len()
|
||
self.eval_wavs: tp.Optional[torch.Tensor] = self._load_eval_wavs(eval_wavs, n_eval_wavs)
|
||
self.cache = None
|
||
if cache_path is not None:
|
||
self.cache = EmbeddingCache(Path(cache_path) / 'wav', self.device,
|
||
compute_embed_fn=self._get_full_chroma_for_cache,
|
||
extract_embed_fn=self._extract_chroma_chunk)
|
||
|
||
def _downsampling_factor(self) -> int:
|
||
return self.chroma.winhop
|
||
|
||
def _load_eval_wavs(self, path: tp.Optional[str], num_samples: int) -> tp.Optional[torch.Tensor]:
|
||
"""Load pre-defined waveforms from a json.
|
||
These waveforms will be used for chroma extraction during evaluation.
|
||
This is done to make the evaluation on MusicCaps fair (we shouldn't see the chromas of MusicCaps).
|
||
"""
|
||
if path is None:
|
||
return None
|
||
|
||
logger.info(f"Loading evaluation wavs from {path}")
|
||
from audiocraft.data.audio_dataset import AudioDataset
|
||
dataset: AudioDataset = AudioDataset.from_meta(
|
||
path, segment_duration=self.duration, min_audio_duration=self.duration,
|
||
sample_rate=self.sample_rate, channels=1)
|
||
|
||
if len(dataset) > 0:
|
||
eval_wavs = dataset.collater([dataset[i] for i in range(num_samples)]).to(self.device)
|
||
logger.info(f"Using {len(eval_wavs)} evaluation wavs for chroma-stem conditioner")
|
||
return eval_wavs
|
||
else:
|
||
raise ValueError("Could not find evaluation wavs, check lengths of wavs")
|
||
|
||
def reset_eval_wavs(self, eval_wavs: tp.Optional[torch.Tensor]) -> None:
|
||
self.eval_wavs = eval_wavs
|
||
|
||
def has_eval_wavs(self) -> bool:
|
||
return self.eval_wavs is not None
|
||
|
||
def _sample_eval_wavs(self, num_samples: int) -> torch.Tensor:
|
||
"""Sample wavs from a predefined list."""
|
||
assert self.eval_wavs is not None, "Cannot sample eval wavs as no eval wavs provided."
|
||
total_eval_wavs = len(self.eval_wavs)
|
||
out = self.eval_wavs
|
||
if num_samples > total_eval_wavs:
|
||
out = self.eval_wavs.repeat(num_samples // total_eval_wavs + 1, 1, 1)
|
||
return out[torch.randperm(len(out))][:num_samples]
|
||
|
||
def _get_chroma_len(self) -> int:
|
||
"""Get length of chroma during training."""
|
||
dummy_wav = torch.zeros((1, int(self.sample_rate * self.duration)), device=self.device)
|
||
dummy_chr = self.chroma(dummy_wav)
|
||
return dummy_chr.shape[1]
|
||
|
||
@torch.no_grad()
|
||
def _get_stemmed_wav(self, wav: torch.Tensor, sample_rate: int) -> torch.Tensor:
|
||
"""Get parts of the wav that holds the melody, extracting the main stems from the wav."""
|
||
from demucs.apply import apply_model
|
||
from demucs.audio import convert_audio
|
||
with self.autocast:
|
||
wav = convert_audio(
|
||
wav, sample_rate, self.demucs.samplerate, self.demucs.audio_channels) # type: ignore
|
||
stems = apply_model(self.demucs, wav, device=self.device) # type: ignore
|
||
stems = stems[:, self.stem_indices] # extract relevant stems for melody conditioning
|
||
mix_wav = stems.sum(1) # merge extracted stems to single waveform
|
||
mix_wav = convert_audio(mix_wav, self.demucs.samplerate, self.sample_rate, 1) # type: ignore
|
||
return mix_wav
|
||
|
||
@torch.no_grad()
|
||
def _extract_chroma(self, wav: torch.Tensor) -> torch.Tensor:
|
||
"""Extract chroma features from the waveform."""
|
||
with self.autocast:
|
||
return self.chroma(wav)
|
||
|
||
@torch.no_grad()
|
||
def _compute_wav_embedding(self, wav: torch.Tensor, sample_rate: int) -> torch.Tensor:
|
||
"""Compute wav embedding, applying stem and chroma extraction."""
|
||
# avoid 0-size tensors when we are working with null conds
|
||
if wav.shape[-1] == 1:
|
||
return self._extract_chroma(wav)
|
||
stems = self._get_stemmed_wav(wav, sample_rate)
|
||
chroma = self._extract_chroma(stems)
|
||
return chroma
|
||
|
||
@torch.no_grad()
|
||
def _get_full_chroma_for_cache(self, path: tp.Union[str, Path], x: WavCondition, idx: int) -> torch.Tensor:
|
||
"""Extract chroma from the whole audio waveform at the given path."""
|
||
wav, sr = audio_read(path)
|
||
wav = wav[None].to(self.device)
|
||
wav = convert_audio(wav, sr, self.sample_rate, to_channels=1)
|
||
chroma = self._compute_wav_embedding(wav, self.sample_rate)[0]
|
||
return chroma
|
||
|
||
def _extract_chroma_chunk(self, full_chroma: torch.Tensor, x: WavCondition, idx: int) -> torch.Tensor:
|
||
"""Extract a chunk of chroma from the full chroma derived from the full waveform."""
|
||
wav_length = x.wav.shape[-1]
|
||
seek_time = x.seek_time[idx]
|
||
assert seek_time is not None, (
|
||
"WavCondition seek_time is required "
|
||
"when extracting chroma chunks from pre-computed chroma.")
|
||
full_chroma = full_chroma.float()
|
||
frame_rate = self.sample_rate / self._downsampling_factor()
|
||
target_length = int(frame_rate * wav_length / self.sample_rate)
|
||
index = int(frame_rate * seek_time)
|
||
out = full_chroma[index: index + target_length]
|
||
out = F.pad(out[None], (0, 0, 0, target_length - out.shape[0]))[0]
|
||
return out.to(self.device)
|
||
|
||
@torch.no_grad()
|
||
def _get_wav_embedding(self, x: WavCondition) -> torch.Tensor:
|
||
"""Get the wav embedding from the WavCondition.
|
||
The conditioner will either extract the embedding on-the-fly computing it from the condition wav directly
|
||
or will rely on the embedding cache to load the pre-computed embedding if relevant.
|
||
"""
|
||
sampled_wav: tp.Optional[torch.Tensor] = None
|
||
if not self.training and self.eval_wavs is not None:
|
||
warn_once(logger, "Using precomputed evaluation wavs!")
|
||
sampled_wav = self._sample_eval_wavs(len(x.wav))
|
||
|
||
no_undefined_paths = all(p is not None for p in x.path)
|
||
no_nullified_cond = x.wav.shape[-1] > 1
|
||
if sampled_wav is not None:
|
||
chroma = self._compute_wav_embedding(sampled_wav, self.sample_rate)
|
||
elif self.cache is not None and no_undefined_paths and no_nullified_cond:
|
||
paths = [Path(p) for p in x.path if p is not None]
|
||
chroma = self.cache.get_embed_from_cache(paths, x)
|
||
else:
|
||
assert all(sr == x.sample_rate[0] for sr in x.sample_rate), "All sample rates in batch should be equal."
|
||
chroma = self._compute_wav_embedding(x.wav, x.sample_rate[0])
|
||
|
||
if self.match_len_on_eval:
|
||
B, T, C = chroma.shape
|
||
if T > self.chroma_len:
|
||
chroma = chroma[:, :self.chroma_len]
|
||
logger.debug(f"Chroma was truncated to match length! ({T} -> {chroma.shape[1]})")
|
||
elif T < self.chroma_len:
|
||
n_repeat = int(math.ceil(self.chroma_len / T))
|
||
chroma = chroma.repeat(1, n_repeat, 1)
|
||
chroma = chroma[:, :self.chroma_len]
|
||
logger.debug(f"Chroma was repeated to match length! ({T} -> {chroma.shape[1]})")
|
||
|
||
return chroma
|
||
|
||
def tokenize(self, x: WavCondition) -> WavCondition:
|
||
"""Apply WavConditioner tokenization and populate cache if needed."""
|
||
x = super().tokenize(x)
|
||
no_undefined_paths = all(p is not None for p in x.path)
|
||
if self.cache is not None and no_undefined_paths:
|
||
paths = [Path(p) for p in x.path if p is not None]
|
||
self.cache.populate_embed_cache(paths, x)
|
||
return x</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Chroma conditioner based on stems.
|
||
The ChromaStemConditioner uses DEMUCS to first filter out drums and bass, as
|
||
the drums and bass often dominate the chroma leading to the chroma features
|
||
not containing information about the melody.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>output_dim</code></strong> : <code>int</code></dt>
|
||
<dd>Output dimension for the conditioner.</dd>
|
||
<dt><strong><code>sample_rate</code></strong> : <code>int</code></dt>
|
||
<dd>Sample rate for the chroma extractor.</dd>
|
||
<dt><strong><code>n_chroma</code></strong> : <code>int</code></dt>
|
||
<dd>Number of chroma bins for the chroma extractor.</dd>
|
||
<dt><strong><code>radix2_exp</code></strong> : <code>int</code></dt>
|
||
<dd>Size of stft window for the chroma extractor (power of 2, e.g. 12 -> 2^12).</dd>
|
||
<dt><strong><code>duration</code></strong> : <code>int</code></dt>
|
||
<dd>duration used during training. This is later used for correct padding
|
||
in case we are using chroma as prefix.</dd>
|
||
<dt><strong><code>match_len_on_eval</code></strong> : <code>bool</code>, optional</dt>
|
||
<dd>if True then all chromas are padded to the training
|
||
duration. Defaults to False.</dd>
|
||
<dt><strong><code>eval_wavs</code></strong> : <code>str</code>, optional</dt>
|
||
<dd>path to a dataset manifest with waveform, this waveforms are used as
|
||
conditions during eval (for cases where we don't want to leak test conditions like MusicCaps).
|
||
Defaults to None.</dd>
|
||
<dt><strong><code>n_eval_wavs</code></strong> : <code>int</code>, optional</dt>
|
||
<dd>limits the number of waveforms used for conditioning. Defaults to 0.</dd>
|
||
<dt><strong><code>device</code></strong> : <code>tp.Union[torch.device, str]</code>, optional</dt>
|
||
<dd>Device for the conditioner.</dd>
|
||
<dt><strong><code>**kwargs</code></strong></dt>
|
||
<dd>Additional parameters for the chroma extractor.</dd>
|
||
</dl>
|
||
<p>Initializes internal Module state, shared by both nn.Module and ScriptModule.</p></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.conditioners.WaveformConditioner" href="#audiocraft.modules.conditioners.WaveformConditioner">WaveformConditioner</a></li>
|
||
<li><a title="audiocraft.modules.conditioners.BaseConditioner" href="#audiocraft.modules.conditioners.BaseConditioner">BaseConditioner</a></li>
|
||
<li>torch.nn.modules.module.Module</li>
|
||
</ul>
|
||
<h3>Class variables</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.ChromaStemConditioner.call_super_init"><code class="name">var <span class="ident">call_super_init</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.ChromaStemConditioner.dump_patches"><code class="name">var <span class="ident">dump_patches</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.ChromaStemConditioner.training"><code class="name">var <span class="ident">training</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
<h3>Methods</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.ChromaStemConditioner.has_eval_wavs"><code class="name flex">
|
||
<span>def <span class="ident">has_eval_wavs</span></span>(<span>self) ‑> bool</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def has_eval_wavs(self) -> bool:
|
||
return self.eval_wavs is not None</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.ChromaStemConditioner.reset_eval_wavs"><code class="name flex">
|
||
<span>def <span class="ident">reset_eval_wavs</span></span>(<span>self, eval_wavs: torch.Tensor | None) ‑> None</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def reset_eval_wavs(self, eval_wavs: tp.Optional[torch.Tensor]) -> None:
|
||
self.eval_wavs = eval_wavs</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.ChromaStemConditioner.tokenize"><code class="name flex">
|
||
<span>def <span class="ident">tokenize</span></span>(<span>self,<br>x: <a title="audiocraft.modules.conditioners.WavCondition" href="#audiocraft.modules.conditioners.WavCondition">WavCondition</a>) ‑> <a title="audiocraft.modules.conditioners.WavCondition" href="#audiocraft.modules.conditioners.WavCondition">WavCondition</a></span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def tokenize(self, x: WavCondition) -> WavCondition:
|
||
"""Apply WavConditioner tokenization and populate cache if needed."""
|
||
x = super().tokenize(x)
|
||
no_undefined_paths = all(p is not None for p in x.path)
|
||
if self.cache is not None and no_undefined_paths:
|
||
paths = [Path(p) for p in x.path if p is not None]
|
||
self.cache.populate_embed_cache(paths, x)
|
||
return x</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Apply WavConditioner tokenization and populate cache if needed.</p></div>
|
||
</dd>
|
||
</dl>
|
||
<h3>Inherited members</h3>
|
||
<ul class="hlist">
|
||
<li><code><b><a title="audiocraft.modules.conditioners.WaveformConditioner" href="#audiocraft.modules.conditioners.WaveformConditioner">WaveformConditioner</a></b></code>:
|
||
<ul class="hlist">
|
||
<li><code><a title="audiocraft.modules.conditioners.WaveformConditioner.forward" href="#audiocraft.modules.conditioners.WaveformConditioner.forward">forward</a></code></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.ClassifierFreeGuidanceDropout"><code class="flex name class">
|
||
<span>class <span class="ident">ClassifierFreeGuidanceDropout</span></span>
|
||
<span>(</span><span>p: float, seed: int = 1234)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class ClassifierFreeGuidanceDropout(DropoutModule):
|
||
"""Classifier Free Guidance dropout.
|
||
All attributes are dropped with the same probability.
|
||
|
||
Args:
|
||
p (float): Probability to apply condition dropout during training.
|
||
seed (int): Random seed.
|
||
"""
|
||
def __init__(self, p: float, seed: int = 1234):
|
||
super().__init__(seed=seed)
|
||
self.p = p
|
||
|
||
def forward(self, samples: tp.List[ConditioningAttributes],
|
||
cond_types: tp.List[str] = ["wav", "text"],
|
||
**kwargs) -> tp.List[ConditioningAttributes]:
|
||
"""
|
||
Args:
|
||
samples (list[ConditioningAttributes]): List of conditions.
|
||
Returns:
|
||
list[ConditioningAttributes]: List of conditions after all attributes were set to None.
|
||
"""
|
||
if not self.training:
|
||
return samples
|
||
|
||
# decide on which attributes to drop in a batched fashion
|
||
drop = torch.rand(1, generator=self.rng).item() < self.p
|
||
if not drop:
|
||
return samples
|
||
|
||
# nullify conditions of all attributes
|
||
samples = deepcopy(samples)
|
||
for condition_type in cond_types:
|
||
for sample in samples:
|
||
for condition in sample.attributes[condition_type]:
|
||
dropout_condition(sample, condition_type, condition,
|
||
**kwargs)
|
||
return samples
|
||
|
||
def __repr__(self):
|
||
return f"ClassifierFreeGuidanceDropout(p={self.p})"</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Classifier Free Guidance dropout.
|
||
All attributes are dropped with the same probability.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>p</code></strong> : <code>float</code></dt>
|
||
<dd>Probability to apply condition dropout during training.</dd>
|
||
<dt><strong><code>seed</code></strong> : <code>int</code></dt>
|
||
<dd>Random seed.</dd>
|
||
</dl>
|
||
<p>Initializes internal Module state, shared by both nn.Module and ScriptModule.</p></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.conditioners.DropoutModule" href="#audiocraft.modules.conditioners.DropoutModule">DropoutModule</a></li>
|
||
<li>torch.nn.modules.module.Module</li>
|
||
</ul>
|
||
<h3>Class variables</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.ClassifierFreeGuidanceDropout.call_super_init"><code class="name">var <span class="ident">call_super_init</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.ClassifierFreeGuidanceDropout.dump_patches"><code class="name">var <span class="ident">dump_patches</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.ClassifierFreeGuidanceDropout.training"><code class="name">var <span class="ident">training</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
<h3>Methods</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.ClassifierFreeGuidanceDropout.forward"><code class="name flex">
|
||
<span>def <span class="ident">forward</span></span>(<span>self,<br>samples: List[<a title="audiocraft.modules.conditioners.ConditioningAttributes" href="#audiocraft.modules.conditioners.ConditioningAttributes">ConditioningAttributes</a>],<br>cond_types: List[str] = ['wav', 'text'],<br>**kwargs) ‑> List[<a title="audiocraft.modules.conditioners.ConditioningAttributes" href="#audiocraft.modules.conditioners.ConditioningAttributes">ConditioningAttributes</a>]</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def forward(self, samples: tp.List[ConditioningAttributes],
|
||
cond_types: tp.List[str] = ["wav", "text"],
|
||
**kwargs) -> tp.List[ConditioningAttributes]:
|
||
"""
|
||
Args:
|
||
samples (list[ConditioningAttributes]): List of conditions.
|
||
Returns:
|
||
list[ConditioningAttributes]: List of conditions after all attributes were set to None.
|
||
"""
|
||
if not self.training:
|
||
return samples
|
||
|
||
# decide on which attributes to drop in a batched fashion
|
||
drop = torch.rand(1, generator=self.rng).item() < self.p
|
||
if not drop:
|
||
return samples
|
||
|
||
# nullify conditions of all attributes
|
||
samples = deepcopy(samples)
|
||
for condition_type in cond_types:
|
||
for sample in samples:
|
||
for condition in sample.attributes[condition_type]:
|
||
dropout_condition(sample, condition_type, condition,
|
||
**kwargs)
|
||
return samples</code></pre>
|
||
</details>
|
||
<div class="desc"><h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>samples</code></strong> : <code>list[<a title="audiocraft.modules.conditioners.ConditioningAttributes" href="#audiocraft.modules.conditioners.ConditioningAttributes">ConditioningAttributes</a>]</code></dt>
|
||
<dd>List of conditions.</dd>
|
||
</dl>
|
||
<h2 id="returns">Returns</h2>
|
||
<dl>
|
||
<dt><code>list[<a title="audiocraft.modules.conditioners.ConditioningAttributes" href="#audiocraft.modules.conditioners.ConditioningAttributes">ConditioningAttributes</a>]</code></dt>
|
||
<dd>List of conditions after all attributes were set to None.</dd>
|
||
</dl></div>
|
||
</dd>
|
||
</dl>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.ConditionFuser"><code class="flex name class">
|
||
<span>class <span class="ident">ConditionFuser</span></span>
|
||
<span>(</span><span>fuse2cond: Dict[str, List[str]],<br>cross_attention_pos_emb: bool = False,<br>cross_attention_pos_emb_scale: float = 1.0)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class ConditionFuser(StreamingModule):
|
||
"""Condition fuser handles the logic to combine the different conditions
|
||
to the actual model input.
|
||
|
||
Args:
|
||
fuse2cond (tp.Dict[str, str]): A dictionary that says how to fuse
|
||
each condition. For example:
|
||
{
|
||
"prepend": ["description"],
|
||
"sum": ["genre", "bpm"],
|
||
"cross": ["description"],
|
||
}
|
||
cross_attention_pos_emb (bool, optional): Use positional embeddings in cross attention.
|
||
cross_attention_pos_emb_scale (int): Scale for positional embeddings in cross attention if used.
|
||
"""
|
||
FUSING_METHODS = ["sum", "prepend", "cross", "ignore", "input_interpolate"]
|
||
|
||
def __init__(self, fuse2cond: tp.Dict[str, tp.List[str]], cross_attention_pos_emb: bool = False,
|
||
cross_attention_pos_emb_scale: float = 1.0):
|
||
super().__init__()
|
||
assert all(
|
||
[k in self.FUSING_METHODS for k in fuse2cond.keys()]
|
||
), f"Got invalid fuse method, allowed methods: {self.FUSING_METHODS}"
|
||
self.cross_attention_pos_emb = cross_attention_pos_emb
|
||
self.cross_attention_pos_emb_scale = cross_attention_pos_emb_scale
|
||
self.fuse2cond: tp.Dict[str, tp.List[str]] = fuse2cond
|
||
self.cond2fuse: tp.Dict[str, str] = {}
|
||
for fuse_method, conditions in fuse2cond.items():
|
||
for condition in conditions:
|
||
self.cond2fuse[condition] = fuse_method
|
||
|
||
def forward(
|
||
self,
|
||
input: torch.Tensor,
|
||
conditions: tp.Dict[str, ConditionType]
|
||
) -> tp.Tuple[torch.Tensor, tp.Optional[torch.Tensor]]:
|
||
"""Fuse the conditions to the provided model input.
|
||
|
||
Args:
|
||
input (torch.Tensor): Transformer input.
|
||
conditions (dict[str, ConditionType]): Dict of conditions.
|
||
Returns:
|
||
tuple[torch.Tensor, torch.Tensor]: The first tensor is the transformer input
|
||
after the conditions have been fused. The second output tensor is the tensor
|
||
used for cross-attention or None if no cross attention inputs exist.
|
||
"""
|
||
B, T, _ = input.shape
|
||
|
||
if 'offsets' in self._streaming_state:
|
||
first_step = False
|
||
offsets = self._streaming_state['offsets']
|
||
else:
|
||
first_step = True
|
||
offsets = torch.zeros(input.shape[0], dtype=torch.long, device=input.device)
|
||
|
||
assert set(conditions.keys()).issubset(set(self.cond2fuse.keys())), \
|
||
f"given conditions contain unknown attributes for fuser, " \
|
||
f"expected {self.cond2fuse.keys()}, got {conditions.keys()}"
|
||
cross_attention_output = None
|
||
for cond_type, (cond, cond_mask) in conditions.items():
|
||
op = self.cond2fuse[cond_type]
|
||
if op == 'sum':
|
||
input += cond
|
||
elif op == 'input_interpolate':
|
||
cond = einops.rearrange(cond, "b t d -> b d t")
|
||
cond = F.interpolate(cond, size=input.shape[1])
|
||
input += einops.rearrange(cond, "b d t -> b t d")
|
||
elif op == 'prepend':
|
||
if first_step:
|
||
input = torch.cat([cond, input], dim=1)
|
||
elif op == 'cross':
|
||
if cross_attention_output is not None:
|
||
cross_attention_output = torch.cat([cross_attention_output, cond], dim=1)
|
||
else:
|
||
cross_attention_output = cond
|
||
elif op == 'ignore':
|
||
continue
|
||
else:
|
||
raise ValueError(f"unknown op ({op})")
|
||
|
||
if self.cross_attention_pos_emb and cross_attention_output is not None:
|
||
positions = torch.arange(
|
||
cross_attention_output.shape[1],
|
||
device=cross_attention_output.device
|
||
).view(1, -1, 1)
|
||
pos_emb = create_sin_embedding(positions, cross_attention_output.shape[-1])
|
||
cross_attention_output = cross_attention_output + self.cross_attention_pos_emb_scale * pos_emb
|
||
|
||
if self._is_streaming:
|
||
self._streaming_state['offsets'] = offsets + T
|
||
|
||
return input, cross_attention_output</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Condition fuser handles the logic to combine the different conditions
|
||
to the actual model input.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>fuse2cond</code></strong> : <code>tp.Dict[str, str]</code></dt>
|
||
<dd>A dictionary that says how to fuse
|
||
each condition. For example:
|
||
{
|
||
"prepend": ["description"],
|
||
"sum": ["genre", "bpm"],
|
||
"cross": ["description"],
|
||
}</dd>
|
||
<dt><strong><code>cross_attention_pos_emb</code></strong> : <code>bool</code>, optional</dt>
|
||
<dd>Use positional embeddings in cross attention.</dd>
|
||
<dt><strong><code>cross_attention_pos_emb_scale</code></strong> : <code>int</code></dt>
|
||
<dd>Scale for positional embeddings in cross attention if used.</dd>
|
||
</dl>
|
||
<p>Initializes internal Module state, shared by both nn.Module and ScriptModule.</p></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.streaming.StreamingModule" href="streaming.html#audiocraft.modules.streaming.StreamingModule">StreamingModule</a></li>
|
||
<li>torch.nn.modules.module.Module</li>
|
||
</ul>
|
||
<h3>Class variables</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.ConditionFuser.FUSING_METHODS"><code class="name">var <span class="ident">FUSING_METHODS</span></code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.ConditionFuser.call_super_init"><code class="name">var <span class="ident">call_super_init</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.ConditionFuser.dump_patches"><code class="name">var <span class="ident">dump_patches</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.ConditionFuser.training"><code class="name">var <span class="ident">training</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
<h3>Methods</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.ConditionFuser.forward"><code class="name flex">
|
||
<span>def <span class="ident">forward</span></span>(<span>self,<br>input: torch.Tensor,<br>conditions: Dict[str, Tuple[torch.Tensor, torch.Tensor]]) ‑> Tuple[torch.Tensor, torch.Tensor | None]</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def forward(
|
||
self,
|
||
input: torch.Tensor,
|
||
conditions: tp.Dict[str, ConditionType]
|
||
) -> tp.Tuple[torch.Tensor, tp.Optional[torch.Tensor]]:
|
||
"""Fuse the conditions to the provided model input.
|
||
|
||
Args:
|
||
input (torch.Tensor): Transformer input.
|
||
conditions (dict[str, ConditionType]): Dict of conditions.
|
||
Returns:
|
||
tuple[torch.Tensor, torch.Tensor]: The first tensor is the transformer input
|
||
after the conditions have been fused. The second output tensor is the tensor
|
||
used for cross-attention or None if no cross attention inputs exist.
|
||
"""
|
||
B, T, _ = input.shape
|
||
|
||
if 'offsets' in self._streaming_state:
|
||
first_step = False
|
||
offsets = self._streaming_state['offsets']
|
||
else:
|
||
first_step = True
|
||
offsets = torch.zeros(input.shape[0], dtype=torch.long, device=input.device)
|
||
|
||
assert set(conditions.keys()).issubset(set(self.cond2fuse.keys())), \
|
||
f"given conditions contain unknown attributes for fuser, " \
|
||
f"expected {self.cond2fuse.keys()}, got {conditions.keys()}"
|
||
cross_attention_output = None
|
||
for cond_type, (cond, cond_mask) in conditions.items():
|
||
op = self.cond2fuse[cond_type]
|
||
if op == 'sum':
|
||
input += cond
|
||
elif op == 'input_interpolate':
|
||
cond = einops.rearrange(cond, "b t d -> b d t")
|
||
cond = F.interpolate(cond, size=input.shape[1])
|
||
input += einops.rearrange(cond, "b d t -> b t d")
|
||
elif op == 'prepend':
|
||
if first_step:
|
||
input = torch.cat([cond, input], dim=1)
|
||
elif op == 'cross':
|
||
if cross_attention_output is not None:
|
||
cross_attention_output = torch.cat([cross_attention_output, cond], dim=1)
|
||
else:
|
||
cross_attention_output = cond
|
||
elif op == 'ignore':
|
||
continue
|
||
else:
|
||
raise ValueError(f"unknown op ({op})")
|
||
|
||
if self.cross_attention_pos_emb and cross_attention_output is not None:
|
||
positions = torch.arange(
|
||
cross_attention_output.shape[1],
|
||
device=cross_attention_output.device
|
||
).view(1, -1, 1)
|
||
pos_emb = create_sin_embedding(positions, cross_attention_output.shape[-1])
|
||
cross_attention_output = cross_attention_output + self.cross_attention_pos_emb_scale * pos_emb
|
||
|
||
if self._is_streaming:
|
||
self._streaming_state['offsets'] = offsets + T
|
||
|
||
return input, cross_attention_output</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Fuse the conditions to the provided model input.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>input</code></strong> : <code>torch.Tensor</code></dt>
|
||
<dd>Transformer input.</dd>
|
||
<dt><strong><code>conditions</code></strong> : <code>dict[str, ConditionType]</code></dt>
|
||
<dd>Dict of conditions.</dd>
|
||
</dl>
|
||
<h2 id="returns">Returns</h2>
|
||
<dl>
|
||
<dt><code>tuple[torch.Tensor, torch.Tensor]</code></dt>
|
||
<dd>The first tensor is the transformer input
|
||
after the conditions have been fused. The second output tensor is the tensor
|
||
used for cross-attention or None if no cross attention inputs exist.</dd>
|
||
</dl></div>
|
||
</dd>
|
||
</dl>
|
||
<h3>Inherited members</h3>
|
||
<ul class="hlist">
|
||
<li><code><b><a title="audiocraft.modules.streaming.StreamingModule" href="streaming.html#audiocraft.modules.streaming.StreamingModule">StreamingModule</a></b></code>:
|
||
<ul class="hlist">
|
||
<li><code><a title="audiocraft.modules.streaming.StreamingModule.flush" href="streaming.html#audiocraft.modules.streaming.StreamingModule.flush">flush</a></code></li>
|
||
<li><code><a title="audiocraft.modules.streaming.StreamingModule.get_streaming_state" href="streaming.html#audiocraft.modules.streaming.StreamingModule.get_streaming_state">get_streaming_state</a></code></li>
|
||
<li><code><a title="audiocraft.modules.streaming.StreamingModule.reset_streaming" href="streaming.html#audiocraft.modules.streaming.StreamingModule.reset_streaming">reset_streaming</a></code></li>
|
||
<li><code><a title="audiocraft.modules.streaming.StreamingModule.set_streaming_state" href="streaming.html#audiocraft.modules.streaming.StreamingModule.set_streaming_state">set_streaming_state</a></code></li>
|
||
<li><code><a title="audiocraft.modules.streaming.StreamingModule.streaming" href="streaming.html#audiocraft.modules.streaming.StreamingModule.streaming">streaming</a></code></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.ConditioningAttributes"><code class="flex name class">
|
||
<span>class <span class="ident">ConditioningAttributes</span></span>
|
||
<span>(</span><span>text: Dict[str, str | None] = <factory>,<br>wav: Dict[str, <a title="audiocraft.modules.conditioners.WavCondition" href="#audiocraft.modules.conditioners.WavCondition">WavCondition</a>] = <factory>,<br>joint_embed: Dict[str, <a title="audiocraft.modules.conditioners.JointEmbedCondition" href="#audiocraft.modules.conditioners.JointEmbedCondition">JointEmbedCondition</a>] = <factory>,<br>symbolic: Dict[str, <a title="audiocraft.modules.conditioners.SymbolicCondition" href="#audiocraft.modules.conditioners.SymbolicCondition">SymbolicCondition</a>] = <factory>)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">@dataclass
|
||
class ConditioningAttributes:
|
||
text: tp.Dict[str, tp.Optional[str]] = field(default_factory=dict)
|
||
wav: tp.Dict[str, WavCondition] = field(default_factory=dict)
|
||
joint_embed: tp.Dict[str, JointEmbedCondition] = field(default_factory=dict)
|
||
symbolic: tp.Dict[str, SymbolicCondition] = field(default_factory=dict)
|
||
|
||
def __getitem__(self, item):
|
||
return getattr(self, item)
|
||
|
||
@property
|
||
def text_attributes(self):
|
||
return self.text.keys()
|
||
|
||
@property
|
||
def wav_attributes(self):
|
||
return self.wav.keys()
|
||
|
||
@property
|
||
def joint_embed_attributes(self):
|
||
return self.joint_embed.keys()
|
||
|
||
@property
|
||
def symbolic_attributes(self):
|
||
return self.symbolic.keys()
|
||
|
||
@property
|
||
def attributes(self):
|
||
return {
|
||
"text": self.text_attributes,
|
||
"wav": self.wav_attributes,
|
||
"joint_embed": self.joint_embed_attributes,
|
||
"symbolic": self.symbolic_attributes,
|
||
}
|
||
|
||
def to_flat_dict(self):
|
||
return {
|
||
**{f"text.{k}": v for k, v in self.text.items()},
|
||
**{f"wav.{k}": v for k, v in self.wav.items()},
|
||
**{f"joint_embed.{k}": v for k, v in self.joint_embed.items()},
|
||
**{f"symbolic.{k}": v for k, v in self.symbolic.items()}
|
||
}
|
||
|
||
@classmethod
|
||
def from_flat_dict(cls, x):
|
||
out = cls()
|
||
for k, v in x.items():
|
||
kind, att = k.split(".")
|
||
out[kind][att] = v
|
||
return out</code></pre>
|
||
</details>
|
||
<div class="desc"><p>ConditioningAttributes(text: Dict[str, Optional[str]] = <factory>, wav: Dict[str, audiocraft.modules.conditioners.WavCondition] = <factory>, joint_embed: Dict[str, audiocraft.modules.conditioners.JointEmbedCondition] = <factory>, symbolic: Dict[str, audiocraft.modules.conditioners.SymbolicCondition] = <factory>)</p></div>
|
||
<h3>Class variables</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.ConditioningAttributes.joint_embed"><code class="name">var <span class="ident">joint_embed</span> : Dict[str, <a title="audiocraft.modules.conditioners.JointEmbedCondition" href="#audiocraft.modules.conditioners.JointEmbedCondition">JointEmbedCondition</a>]</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.ConditioningAttributes.symbolic"><code class="name">var <span class="ident">symbolic</span> : Dict[str, <a title="audiocraft.modules.conditioners.SymbolicCondition" href="#audiocraft.modules.conditioners.SymbolicCondition">SymbolicCondition</a>]</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.ConditioningAttributes.text"><code class="name">var <span class="ident">text</span> : Dict[str, str | None]</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.ConditioningAttributes.wav"><code class="name">var <span class="ident">wav</span> : Dict[str, <a title="audiocraft.modules.conditioners.WavCondition" href="#audiocraft.modules.conditioners.WavCondition">WavCondition</a>]</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
<h3>Static methods</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.ConditioningAttributes.from_flat_dict"><code class="name flex">
|
||
<span>def <span class="ident">from_flat_dict</span></span>(<span>x)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
<h3>Instance variables</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.ConditioningAttributes.attributes"><code class="name">prop <span class="ident">attributes</span></code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">@property
|
||
def attributes(self):
|
||
return {
|
||
"text": self.text_attributes,
|
||
"wav": self.wav_attributes,
|
||
"joint_embed": self.joint_embed_attributes,
|
||
"symbolic": self.symbolic_attributes,
|
||
}</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.ConditioningAttributes.joint_embed_attributes"><code class="name">prop <span class="ident">joint_embed_attributes</span></code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">@property
|
||
def joint_embed_attributes(self):
|
||
return self.joint_embed.keys()</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.ConditioningAttributes.symbolic_attributes"><code class="name">prop <span class="ident">symbolic_attributes</span></code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">@property
|
||
def symbolic_attributes(self):
|
||
return self.symbolic.keys()</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.ConditioningAttributes.text_attributes"><code class="name">prop <span class="ident">text_attributes</span></code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">@property
|
||
def text_attributes(self):
|
||
return self.text.keys()</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.ConditioningAttributes.wav_attributes"><code class="name">prop <span class="ident">wav_attributes</span></code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">@property
|
||
def wav_attributes(self):
|
||
return self.wav.keys()</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
<h3>Methods</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.ConditioningAttributes.to_flat_dict"><code class="name flex">
|
||
<span>def <span class="ident">to_flat_dict</span></span>(<span>self)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def to_flat_dict(self):
|
||
return {
|
||
**{f"text.{k}": v for k, v in self.text.items()},
|
||
**{f"wav.{k}": v for k, v in self.wav.items()},
|
||
**{f"joint_embed.{k}": v for k, v in self.joint_embed.items()},
|
||
**{f"symbolic.{k}": v for k, v in self.symbolic.items()}
|
||
}</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.ConditioningProvider"><code class="flex name class">
|
||
<span>class <span class="ident">ConditioningProvider</span></span>
|
||
<span>(</span><span>conditioners: Dict[str, <a title="audiocraft.modules.conditioners.BaseConditioner" href="#audiocraft.modules.conditioners.BaseConditioner">BaseConditioner</a>],<br>device: torch.device | str = 'cpu')</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class ConditioningProvider(nn.Module):
|
||
"""Prepare and provide conditions given all the supported conditioners.
|
||
|
||
Args:
|
||
conditioners (dict): Dictionary of conditioners.
|
||
device (torch.device or str, optional): Device for conditioners and output condition types.
|
||
"""
|
||
def __init__(self, conditioners: tp.Dict[str, BaseConditioner], device: tp.Union[torch.device, str] = "cpu"):
|
||
super().__init__()
|
||
self.device = device
|
||
self.conditioners = nn.ModuleDict(conditioners)
|
||
|
||
@property
|
||
def joint_embed_conditions(self):
|
||
return [m.attribute for m in self.conditioners.values() if isinstance(m, JointEmbeddingConditioner)]
|
||
|
||
@property
|
||
def has_joint_embed_conditions(self):
|
||
return len(self.joint_embed_conditions) > 0
|
||
|
||
@property
|
||
def text_conditions(self):
|
||
return [k for k, v in self.conditioners.items() if isinstance(v, TextConditioner)]
|
||
|
||
@property
|
||
def wav_conditions(self):
|
||
return [k for k, v in self.conditioners.items() if isinstance(v, WaveformConditioner)]
|
||
|
||
@property
|
||
def has_wav_condition(self):
|
||
return len(self.wav_conditions) > 0
|
||
|
||
def tokenize(self, inputs: tp.List[ConditioningAttributes]) -> tp.Dict[str, tp.Any]:
|
||
"""Match attributes/wavs with existing conditioners in self, and compute tokenize them accordingly.
|
||
This should be called before starting any real GPU work to avoid synchronization points.
|
||
This will return a dict matching conditioner names to their arbitrary tokenized representations.
|
||
|
||
Args:
|
||
inputs (list[ConditioningAttributes]): List of ConditioningAttributes objects containing
|
||
text and wav conditions.
|
||
"""
|
||
assert all([isinstance(x, ConditioningAttributes) for x in inputs]), (
|
||
"Got unexpected types input for conditioner! should be tp.List[ConditioningAttributes]",
|
||
f" but types were {set([type(x) for x in inputs])}"
|
||
)
|
||
|
||
output = {}
|
||
text = self._collate_text(inputs)
|
||
wavs = self._collate_wavs(inputs)
|
||
joint_embeds = self._collate_joint_embeds(inputs)
|
||
|
||
assert set(text.keys() | wavs.keys() | joint_embeds.keys()).issubset(set(self.conditioners.keys())), (
|
||
f"Got an unexpected attribute! Expected {self.conditioners.keys()}, ",
|
||
f"got {text.keys(), wavs.keys(), joint_embeds.keys()}"
|
||
)
|
||
|
||
for attribute, batch in chain(text.items(), wavs.items(), joint_embeds.items()):
|
||
output[attribute] = self.conditioners[attribute].tokenize(batch)
|
||
return output
|
||
|
||
def forward(self, tokenized: tp.Dict[str, tp.Any]) -> tp.Dict[str, ConditionType]:
|
||
"""Compute pairs of `(embedding, mask)` using the configured conditioners and the tokenized representations.
|
||
The output is for example:
|
||
{
|
||
"genre": (torch.Tensor([B, 1, D_genre]), torch.Tensor([B, 1])),
|
||
"description": (torch.Tensor([B, T_desc, D_desc]), torch.Tensor([B, T_desc])),
|
||
...
|
||
}
|
||
|
||
Args:
|
||
tokenized (dict): Dict of tokenized representations as returned by `tokenize()`.
|
||
"""
|
||
output = {}
|
||
for attribute, inputs in tokenized.items():
|
||
condition, mask = self.conditioners[attribute](inputs)
|
||
output[attribute] = (condition, mask)
|
||
return output
|
||
|
||
def _collate_text(self, samples: tp.List[ConditioningAttributes]) -> tp.Dict[str, tp.List[tp.Optional[str]]]:
|
||
"""Given a list of ConditioningAttributes objects, compile a dictionary where the keys
|
||
are the attributes and the values are the aggregated input per attribute.
|
||
For example:
|
||
Input:
|
||
[
|
||
ConditioningAttributes(text={"genre": "Rock", "description": "A rock song with a guitar solo"}, wav=...),
|
||
ConditioningAttributes(text={"genre": "Hip-hop", "description": "A hip-hop verse"}, wav=...),
|
||
]
|
||
Output:
|
||
{
|
||
"genre": ["Rock", "Hip-hop"],
|
||
"description": ["A rock song with a guitar solo", "A hip-hop verse"]
|
||
}
|
||
|
||
Args:
|
||
samples (list of ConditioningAttributes): List of ConditioningAttributes samples.
|
||
Returns:
|
||
dict[str, list[str, optional]]: A dictionary mapping an attribute name to text batch.
|
||
"""
|
||
out: tp.Dict[str, tp.List[tp.Optional[str]]] = defaultdict(list)
|
||
texts = [x.text for x in samples]
|
||
for text in texts:
|
||
for condition in self.text_conditions:
|
||
out[condition].append(text[condition])
|
||
return out
|
||
|
||
def _collate_wavs(self, samples: tp.List[ConditioningAttributes]) -> tp.Dict[str, WavCondition]:
|
||
"""Generate a dict where the keys are attributes by which we fetch similar wavs,
|
||
and the values are Tensors of wavs according to said attributes.
|
||
|
||
*Note*: by the time the samples reach this function, each sample should have some waveform
|
||
inside the "wav" attribute. It should be either:
|
||
1. A real waveform
|
||
2. A null waveform due to the sample having no similar waveforms (nullified by the dataset)
|
||
3. A null waveform due to it being dropped in a dropout module (nullified by dropout)
|
||
|
||
Args:
|
||
samples (list of ConditioningAttributes): List of ConditioningAttributes samples.
|
||
Returns:
|
||
dict[str, WavCondition]: A dictionary mapping an attribute name to wavs.
|
||
"""
|
||
wavs = defaultdict(list)
|
||
lengths = defaultdict(list)
|
||
sample_rates = defaultdict(list)
|
||
paths = defaultdict(list)
|
||
seek_times = defaultdict(list)
|
||
out: tp.Dict[str, WavCondition] = {}
|
||
|
||
for sample in samples:
|
||
for attribute in self.wav_conditions:
|
||
wav, length, sample_rate, path, seek_time = sample.wav[attribute]
|
||
assert wav.dim() == 3, f"Got wav with dim={wav.dim()}, but expected 3 [1, C, T]"
|
||
assert wav.size(0) == 1, f"Got wav [B, C, T] with shape={wav.shape}, but expected B == 1"
|
||
# mono-channel conditioning
|
||
wav = wav.mean(1, keepdim=True) # [1, 1, T]
|
||
wavs[attribute].append(wav.flatten()) # [T]
|
||
lengths[attribute].append(length)
|
||
sample_rates[attribute].extend(sample_rate)
|
||
paths[attribute].extend(path)
|
||
seek_times[attribute].extend(seek_time)
|
||
|
||
# stack all wavs to a single tensor
|
||
for attribute in self.wav_conditions:
|
||
stacked_wav, _ = collate(wavs[attribute], dim=0)
|
||
out[attribute] = WavCondition(
|
||
stacked_wav.unsqueeze(1), torch.cat(lengths[attribute]), sample_rates[attribute],
|
||
paths[attribute], seek_times[attribute])
|
||
|
||
return out
|
||
|
||
def _collate_joint_embeds(self, samples: tp.List[ConditioningAttributes]) -> tp.Dict[str, JointEmbedCondition]:
|
||
"""Generate a dict where the keys are attributes by which we compute joint embeddings,
|
||
and the values are Tensors of pre-computed embeddings and the corresponding text attributes.
|
||
|
||
Args:
|
||
samples (list[ConditioningAttributes]): List of ConditioningAttributes samples.
|
||
Returns:
|
||
A dictionary mapping an attribute name to joint embeddings.
|
||
"""
|
||
texts = defaultdict(list)
|
||
wavs = defaultdict(list)
|
||
lengths = defaultdict(list)
|
||
sample_rates = defaultdict(list)
|
||
paths = defaultdict(list)
|
||
seek_times = defaultdict(list)
|
||
channels: int = 0
|
||
|
||
out = {}
|
||
for sample in samples:
|
||
for attribute in self.joint_embed_conditions:
|
||
wav, text, length, sample_rate, path, seek_time = sample.joint_embed[attribute]
|
||
assert wav.dim() == 3
|
||
if channels == 0:
|
||
channels = wav.size(1)
|
||
else:
|
||
assert channels == wav.size(1), "not all audio has same number of channels in batch"
|
||
assert wav.size(0) == 1, "Expecting single-wav batch in the collate method"
|
||
wav = einops.rearrange(wav, "b c t -> (b c t)") # [1, C, T] => [C * T]
|
||
wavs[attribute].append(wav)
|
||
texts[attribute].extend(text)
|
||
lengths[attribute].append(length)
|
||
sample_rates[attribute].extend(sample_rate)
|
||
paths[attribute].extend(path)
|
||
seek_times[attribute].extend(seek_time)
|
||
|
||
for attribute in self.joint_embed_conditions:
|
||
stacked_texts = texts[attribute]
|
||
stacked_paths = paths[attribute]
|
||
stacked_seek_times = seek_times[attribute]
|
||
stacked_wavs = pad_sequence(wavs[attribute]).to(self.device)
|
||
stacked_wavs = einops.rearrange(stacked_wavs, "(c t) b -> b c t", c=channels)
|
||
stacked_sample_rates = sample_rates[attribute]
|
||
stacked_lengths = torch.cat(lengths[attribute]).to(self.device)
|
||
assert stacked_lengths.size(0) == stacked_wavs.size(0)
|
||
assert len(stacked_sample_rates) == stacked_wavs.size(0)
|
||
assert len(stacked_texts) == stacked_wavs.size(0)
|
||
out[attribute] = JointEmbedCondition(
|
||
text=stacked_texts, wav=stacked_wavs,
|
||
length=stacked_lengths, sample_rate=stacked_sample_rates,
|
||
path=stacked_paths, seek_time=stacked_seek_times)
|
||
|
||
return out</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Prepare and provide conditions given all the supported conditioners.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>conditioners</code></strong> : <code>dict</code></dt>
|
||
<dd>Dictionary of conditioners.</dd>
|
||
<dt><strong><code>device</code></strong> : <code>torch.device</code> or <code>str</code>, optional</dt>
|
||
<dd>Device for conditioners and output condition types.</dd>
|
||
</dl>
|
||
<p>Initializes internal Module state, shared by both nn.Module and ScriptModule.</p></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li>torch.nn.modules.module.Module</li>
|
||
</ul>
|
||
<h3>Subclasses</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.jasco_conditioners.JascoConditioningProvider" href="jasco_conditioners.html#audiocraft.modules.jasco_conditioners.JascoConditioningProvider">JascoConditioningProvider</a></li>
|
||
</ul>
|
||
<h3>Class variables</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.ConditioningProvider.call_super_init"><code class="name">var <span class="ident">call_super_init</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.ConditioningProvider.dump_patches"><code class="name">var <span class="ident">dump_patches</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.ConditioningProvider.training"><code class="name">var <span class="ident">training</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
<h3>Instance variables</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.ConditioningProvider.has_joint_embed_conditions"><code class="name">prop <span class="ident">has_joint_embed_conditions</span></code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">@property
|
||
def has_joint_embed_conditions(self):
|
||
return len(self.joint_embed_conditions) > 0</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.ConditioningProvider.has_wav_condition"><code class="name">prop <span class="ident">has_wav_condition</span></code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">@property
|
||
def has_wav_condition(self):
|
||
return len(self.wav_conditions) > 0</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.ConditioningProvider.joint_embed_conditions"><code class="name">prop <span class="ident">joint_embed_conditions</span></code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">@property
|
||
def joint_embed_conditions(self):
|
||
return [m.attribute for m in self.conditioners.values() if isinstance(m, JointEmbeddingConditioner)]</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.ConditioningProvider.text_conditions"><code class="name">prop <span class="ident">text_conditions</span></code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">@property
|
||
def text_conditions(self):
|
||
return [k for k, v in self.conditioners.items() if isinstance(v, TextConditioner)]</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.ConditioningProvider.wav_conditions"><code class="name">prop <span class="ident">wav_conditions</span></code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">@property
|
||
def wav_conditions(self):
|
||
return [k for k, v in self.conditioners.items() if isinstance(v, WaveformConditioner)]</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
<h3>Methods</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.ConditioningProvider.forward"><code class="name flex">
|
||
<span>def <span class="ident">forward</span></span>(<span>self, tokenized: Dict[str, Any]) ‑> Dict[str, Tuple[torch.Tensor, torch.Tensor]]</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def forward(self, tokenized: tp.Dict[str, tp.Any]) -> tp.Dict[str, ConditionType]:
|
||
"""Compute pairs of `(embedding, mask)` using the configured conditioners and the tokenized representations.
|
||
The output is for example:
|
||
{
|
||
"genre": (torch.Tensor([B, 1, D_genre]), torch.Tensor([B, 1])),
|
||
"description": (torch.Tensor([B, T_desc, D_desc]), torch.Tensor([B, T_desc])),
|
||
...
|
||
}
|
||
|
||
Args:
|
||
tokenized (dict): Dict of tokenized representations as returned by `tokenize()`.
|
||
"""
|
||
output = {}
|
||
for attribute, inputs in tokenized.items():
|
||
condition, mask = self.conditioners[attribute](inputs)
|
||
output[attribute] = (condition, mask)
|
||
return output</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Compute pairs of <code>(embedding, mask)</code> using the configured conditioners and the tokenized representations.
|
||
The output is for example:
|
||
{
|
||
"genre": (torch.Tensor([B, 1, D_genre]), torch.Tensor([B, 1])),
|
||
"description": (torch.Tensor([B, T_desc, D_desc]), torch.Tensor([B, T_desc])),
|
||
…
|
||
}</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>tokenized</code></strong> : <code>dict</code></dt>
|
||
<dd>Dict of tokenized representations as returned by <code>tokenize()</code>.</dd>
|
||
</dl></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.ConditioningProvider.tokenize"><code class="name flex">
|
||
<span>def <span class="ident">tokenize</span></span>(<span>self,<br>inputs: List[<a title="audiocraft.modules.conditioners.ConditioningAttributes" href="#audiocraft.modules.conditioners.ConditioningAttributes">ConditioningAttributes</a>]) ‑> Dict[str, Any]</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def tokenize(self, inputs: tp.List[ConditioningAttributes]) -> tp.Dict[str, tp.Any]:
|
||
"""Match attributes/wavs with existing conditioners in self, and compute tokenize them accordingly.
|
||
This should be called before starting any real GPU work to avoid synchronization points.
|
||
This will return a dict matching conditioner names to their arbitrary tokenized representations.
|
||
|
||
Args:
|
||
inputs (list[ConditioningAttributes]): List of ConditioningAttributes objects containing
|
||
text and wav conditions.
|
||
"""
|
||
assert all([isinstance(x, ConditioningAttributes) for x in inputs]), (
|
||
"Got unexpected types input for conditioner! should be tp.List[ConditioningAttributes]",
|
||
f" but types were {set([type(x) for x in inputs])}"
|
||
)
|
||
|
||
output = {}
|
||
text = self._collate_text(inputs)
|
||
wavs = self._collate_wavs(inputs)
|
||
joint_embeds = self._collate_joint_embeds(inputs)
|
||
|
||
assert set(text.keys() | wavs.keys() | joint_embeds.keys()).issubset(set(self.conditioners.keys())), (
|
||
f"Got an unexpected attribute! Expected {self.conditioners.keys()}, ",
|
||
f"got {text.keys(), wavs.keys(), joint_embeds.keys()}"
|
||
)
|
||
|
||
for attribute, batch in chain(text.items(), wavs.items(), joint_embeds.items()):
|
||
output[attribute] = self.conditioners[attribute].tokenize(batch)
|
||
return output</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Match attributes/wavs with existing conditioners in self, and compute tokenize them accordingly.
|
||
This should be called before starting any real GPU work to avoid synchronization points.
|
||
This will return a dict matching conditioner names to their arbitrary tokenized representations.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>inputs</code></strong> : <code>list[<a title="audiocraft.modules.conditioners.ConditioningAttributes" href="#audiocraft.modules.conditioners.ConditioningAttributes">ConditioningAttributes</a>]</code></dt>
|
||
<dd>List of ConditioningAttributes objects containing
|
||
text and wav conditions.</dd>
|
||
</dl></div>
|
||
</dd>
|
||
</dl>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.DropoutModule"><code class="flex name class">
|
||
<span>class <span class="ident">DropoutModule</span></span>
|
||
<span>(</span><span>seed: int = 1234)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class DropoutModule(nn.Module):
|
||
"""Base module for all dropout modules."""
|
||
def __init__(self, seed: int = 1234):
|
||
super().__init__()
|
||
self.rng = torch.Generator()
|
||
self.rng.manual_seed(seed)</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Base module for all dropout modules.</p>
|
||
<p>Initializes internal Module state, shared by both nn.Module and ScriptModule.</p></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li>torch.nn.modules.module.Module</li>
|
||
</ul>
|
||
<h3>Subclasses</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.conditioners.AttributeDropout" href="#audiocraft.modules.conditioners.AttributeDropout">AttributeDropout</a></li>
|
||
<li><a title="audiocraft.modules.conditioners.ClassifierFreeGuidanceDropout" href="#audiocraft.modules.conditioners.ClassifierFreeGuidanceDropout">ClassifierFreeGuidanceDropout</a></li>
|
||
</ul>
|
||
<h3>Class variables</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.DropoutModule.call_super_init"><code class="name">var <span class="ident">call_super_init</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.DropoutModule.dump_patches"><code class="name">var <span class="ident">dump_patches</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.DropoutModule.training"><code class="name">var <span class="ident">training</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
<h3>Methods</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.DropoutModule.forward"><code class="name flex">
|
||
<span>def <span class="ident">forward</span></span>(<span>self, *input: Any) ‑> None</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def _forward_unimplemented(self, *input: Any) -> None:
|
||
r"""Defines the computation performed at every call.
|
||
|
||
Should be overridden by all subclasses.
|
||
|
||
.. note::
|
||
Although the recipe for forward pass needs to be defined within
|
||
this function, one should call the :class:`Module` instance afterwards
|
||
instead of this since the former takes care of running the
|
||
registered hooks while the latter silently ignores them.
|
||
"""
|
||
raise NotImplementedError(f"Module [{type(self).__name__}] is missing the required \"forward\" function")</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Defines the computation performed at every call.</p>
|
||
<p>Should be overridden by all subclasses.</p>
|
||
<div class="admonition note">
|
||
<p class="admonition-title">Note</p>
|
||
<p>Although the recipe for forward pass needs to be defined within
|
||
this function, one should call the :class:<code>Module</code> instance afterwards
|
||
instead of this since the former takes care of running the
|
||
registered hooks while the latter silently ignores them.</p>
|
||
</div></div>
|
||
</dd>
|
||
</dl>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.FeatureExtractor"><code class="flex name class">
|
||
<span>class <span class="ident">FeatureExtractor</span></span>
|
||
<span>(</span><span>model_name: str,<br>sample_rate: int,<br>encodec_checkpoint: str,<br>encodec_n_q: int,<br>length: float,<br>dim: int,<br>output_dim: int,<br>device: torch.device | str,<br>compute_mask: bool = True,<br>use_middle_of_segment: bool = False,<br>ds_rate_compression: int = 640,<br>num_codebooks_lm: int = 4)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class FeatureExtractor(WaveformConditioner):
|
||
"""
|
||
Feature Extractor used for the style conditioner of the paper AUDIO CONDITIONING
|
||
FOR MUSIC GENERATION VIA DISCRETE BOTTLENECK FEATURES.
|
||
|
||
Given a waveform, we extract an excerpt of defined length randomly subsampled.
|
||
Then, we feed this excerpt to a feature extractor.
|
||
|
||
Args:
|
||
model_name (str): 'encodec' or 'mert'.
|
||
sample_rate (str): sample rate of the input audio. (32000)
|
||
encodec_checkpoint (str): if encodec is used as a feature extractor, checkpoint
|
||
of the model. ('//pretrained/facebook/encodec_32khz' is the default)
|
||
encodec_n_q (int): if encodec is used as a feature extractor it sets the number of
|
||
quantization streams used in it.
|
||
length (float): length in seconds of the random subsampled excerpt that is used
|
||
for conditioning.
|
||
dim (int): The internal representation dimension.
|
||
output_dim (int): Output dimension for the conditioner.
|
||
device (tp.Union[torch.device, str], optional): Device for the conditioner.
|
||
compute_mask (bool): whether to mask the tokens corresponding to the subsampled
|
||
excerpt in the computation of the music language model cross-entropy loss.
|
||
use_middle_of_segment (bool): if True, always take the middle of the input
|
||
instead of a random subsampled excerpt.
|
||
ds_rate_compression (int): downsampling parameter of the compression model used
|
||
for the music language model. (640 for encodec_32khz)
|
||
num_codebooks_lm (int): the number of codebooks used by the music language model.
|
||
"""
|
||
def __init__(
|
||
self, model_name: str,
|
||
sample_rate: int, encodec_checkpoint: str, encodec_n_q: int, length: float,
|
||
dim: int, output_dim: int, device: tp.Union[torch.device, str],
|
||
compute_mask: bool = True,
|
||
use_middle_of_segment: bool = False, ds_rate_compression: int = 640,
|
||
num_codebooks_lm: int = 4
|
||
):
|
||
assert model_name in ['encodec', 'mert']
|
||
if model_name == 'encodec':
|
||
from ..solvers.compression import CompressionSolver
|
||
feat_extractor = CompressionSolver.model_from_checkpoint(encodec_checkpoint, device)
|
||
elif model_name == 'mert':
|
||
from transformers import AutoModel
|
||
feat_extractor = AutoModel.from_pretrained("m-a-p/MERT-v1-95M", trust_remote_code=True)
|
||
super().__init__(
|
||
dim=dim,
|
||
output_dim=output_dim,
|
||
device=device
|
||
)
|
||
self.sample_rate = sample_rate
|
||
self.compute_mask = compute_mask
|
||
self.feat_extractor: nn.Module
|
||
self.embed: tp.Union[nn.ModuleList, nn.Linear]
|
||
if model_name == 'encodec':
|
||
self.__dict__["feat_extractor"] = feat_extractor.to(device)
|
||
self.encodec_n_q = encodec_n_q
|
||
self.embed = nn.ModuleList([nn.Embedding(feat_extractor.cardinality, dim) for _ in range(encodec_n_q)])
|
||
if model_name == 'mert':
|
||
self.__dict__["feat_extractor"] = feat_extractor.eval().to(device)
|
||
self.embed = nn.Linear(768, dim) # hardcoded
|
||
self.length_subwav = int(length * sample_rate)
|
||
self.ds_rate_compression = ds_rate_compression
|
||
self.model_name = model_name
|
||
self.use_middle_of_segment = use_middle_of_segment
|
||
self.num_codebooks_lm = num_codebooks_lm
|
||
|
||
def _get_wav_embedding(self, x: WavCondition) -> torch.Tensor:
|
||
if x.wav.shape[-1] == 1:
|
||
self.temp_mask = None
|
||
return torch.zeros(x.wav.shape[0], 1, self.dim, device=self.device)
|
||
else:
|
||
with torch.no_grad():
|
||
if self.use_middle_of_segment:
|
||
start = int((x.wav.shape[-1] - self.length_subwav) / 2)
|
||
wav = x.wav[:, :, start:start+self.length_subwav]
|
||
else:
|
||
start = random.randint(0, x.wav.shape[-1] - self.length_subwav)
|
||
wav = x.wav[:, :, start:start+self.length_subwav]
|
||
if self.compute_mask:
|
||
self.temp_mask = self._get_mask_wav(x, start)
|
||
if self.model_name == 'encodec':
|
||
tokens = self.feat_extractor.encode(wav)[0] # type: ignore
|
||
elif self.model_name == 'mert':
|
||
wav = convert_audio(wav, from_rate=x.sample_rate[0], to_rate=24000, to_channels=1)
|
||
embeds = self.feat_extractor(wav.squeeze(-2)).last_hidden_state
|
||
if self.model_name == 'encodec':
|
||
tokens = tokens[:, :self.encodec_n_q]
|
||
embeds = sum([self.embed[k](tokens[:, k]) for k in range(self.encodec_n_q)]) # type: ignore
|
||
else:
|
||
embeds = self.embed(embeds)
|
||
|
||
return embeds # [B, T, dim]
|
||
|
||
def _downsampling_factor(self):
|
||
if self.model_name == 'encodec':
|
||
return self.sample_rate / self.feat_extractor.frame_rate
|
||
elif self.model_name == 'mert':
|
||
return self.sample_rate / 75
|
||
|
||
def _get_mask_wav(self, x: WavCondition, start: int) -> tp.Union[torch.Tensor, None]:
|
||
if x.wav.shape[-1] == 1:
|
||
return None
|
||
total_length = int(x.wav.shape[-1] / self.ds_rate_compression)
|
||
mask_length = int(self.length_subwav / self.ds_rate_compression)
|
||
start = int(start / self.ds_rate_compression)
|
||
mask = torch.ones(x.wav.shape[0], self.num_codebooks_lm,
|
||
total_length, device=self.device, dtype=torch.bool)
|
||
mask[:, :, start:start+mask_length] = 0
|
||
return mask</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Feature Extractor used for the style conditioner of the paper AUDIO CONDITIONING
|
||
FOR MUSIC GENERATION VIA DISCRETE BOTTLENECK FEATURES.</p>
|
||
<p>Given a waveform, we extract an excerpt of defined length randomly subsampled.
|
||
Then, we feed this excerpt to a feature extractor.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>model_name</code></strong> : <code>str</code></dt>
|
||
<dd>'encodec' or 'mert'.</dd>
|
||
<dt><strong><code>sample_rate</code></strong> : <code>str</code></dt>
|
||
<dd>sample rate of the input audio. (32000)</dd>
|
||
<dt><strong><code>encodec_checkpoint</code></strong> : <code>str</code></dt>
|
||
<dd>if encodec is used as a feature extractor, checkpoint
|
||
of the model. ('//pretrained/facebook/encodec_32khz' is the default)</dd>
|
||
<dt><strong><code>encodec_n_q</code></strong> : <code>int</code></dt>
|
||
<dd>if encodec is used as a feature extractor it sets the number of
|
||
quantization streams used in it.</dd>
|
||
<dt><strong><code>length</code></strong> : <code>float</code></dt>
|
||
<dd>length in seconds of the random subsampled excerpt that is used
|
||
for conditioning.</dd>
|
||
<dt><strong><code>dim</code></strong> : <code>int</code></dt>
|
||
<dd>The internal representation dimension.</dd>
|
||
<dt><strong><code>output_dim</code></strong> : <code>int</code></dt>
|
||
<dd>Output dimension for the conditioner.</dd>
|
||
<dt><strong><code>device</code></strong> : <code>tp.Union[torch.device, str]</code>, optional</dt>
|
||
<dd>Device for the conditioner.</dd>
|
||
<dt><strong><code>compute_mask</code></strong> : <code>bool</code></dt>
|
||
<dd>whether to mask the tokens corresponding to the subsampled
|
||
excerpt in the computation of the music language model cross-entropy loss.</dd>
|
||
<dt><strong><code>use_middle_of_segment</code></strong> : <code>bool</code></dt>
|
||
<dd>if True, always take the middle of the input
|
||
instead of a random subsampled excerpt.</dd>
|
||
<dt><strong><code>ds_rate_compression</code></strong> : <code>int</code></dt>
|
||
<dd>downsampling parameter of the compression model used
|
||
for the music language model. (640 for encodec_32khz)</dd>
|
||
<dt><strong><code>num_codebooks_lm</code></strong> : <code>int</code></dt>
|
||
<dd>the number of codebooks used by the music language model.</dd>
|
||
</dl>
|
||
<p>Initializes internal Module state, shared by both nn.Module and ScriptModule.</p></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.conditioners.WaveformConditioner" href="#audiocraft.modules.conditioners.WaveformConditioner">WaveformConditioner</a></li>
|
||
<li><a title="audiocraft.modules.conditioners.BaseConditioner" href="#audiocraft.modules.conditioners.BaseConditioner">BaseConditioner</a></li>
|
||
<li>torch.nn.modules.module.Module</li>
|
||
</ul>
|
||
<h3>Subclasses</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.conditioners.StyleConditioner" href="#audiocraft.modules.conditioners.StyleConditioner">StyleConditioner</a></li>
|
||
</ul>
|
||
<h3>Class variables</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.FeatureExtractor.call_super_init"><code class="name">var <span class="ident">call_super_init</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.FeatureExtractor.dump_patches"><code class="name">var <span class="ident">dump_patches</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.FeatureExtractor.training"><code class="name">var <span class="ident">training</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
<h3>Inherited members</h3>
|
||
<ul class="hlist">
|
||
<li><code><b><a title="audiocraft.modules.conditioners.WaveformConditioner" href="#audiocraft.modules.conditioners.WaveformConditioner">WaveformConditioner</a></b></code>:
|
||
<ul class="hlist">
|
||
<li><code><a title="audiocraft.modules.conditioners.WaveformConditioner.forward" href="#audiocraft.modules.conditioners.WaveformConditioner.forward">forward</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.WaveformConditioner.tokenize" href="#audiocraft.modules.conditioners.BaseConditioner.tokenize">tokenize</a></code></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.JascoCondConst"><code class="flex name class">
|
||
<span>class <span class="ident">JascoCondConst</span></span>
|
||
<span>(</span><span>value, names=None, *, module=None, qualname=None, type=None, start=1)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class JascoCondConst(Enum):
|
||
DRM = 'self_wav'
|
||
CRD = 'chords'
|
||
MLD = 'melody'
|
||
SYM = {'chords', 'melody'}
|
||
LAT = {'self_wav'}
|
||
ALL = ['chords', 'self_wav', 'melody'] # order matters</code></pre>
|
||
</details>
|
||
<div class="desc"><p>An enumeration.</p></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li>enum.Enum</li>
|
||
</ul>
|
||
<h3>Class variables</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.JascoCondConst.ALL"><code class="name">var <span class="ident">ALL</span></code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.JascoCondConst.CRD"><code class="name">var <span class="ident">CRD</span></code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.JascoCondConst.DRM"><code class="name">var <span class="ident">DRM</span></code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.JascoCondConst.LAT"><code class="name">var <span class="ident">LAT</span></code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.JascoCondConst.MLD"><code class="name">var <span class="ident">MLD</span></code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.JascoCondConst.SYM"><code class="name">var <span class="ident">SYM</span></code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.JointEmbedCondition"><code class="flex name class">
|
||
<span>class <span class="ident">JointEmbedCondition</span></span>
|
||
<span>(</span><span>wav: torch.Tensor,<br>text: List[str | None],<br>length: torch.Tensor,<br>sample_rate: List[int],<br>path: List[str | None] = [],<br>seek_time: List[float | None] = [])</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class JointEmbedCondition(tp.NamedTuple):
|
||
wav: torch.Tensor
|
||
text: tp.List[tp.Optional[str]]
|
||
length: torch.Tensor
|
||
sample_rate: tp.List[int]
|
||
path: tp.List[tp.Optional[str]] = []
|
||
seek_time: tp.List[tp.Optional[float]] = []</code></pre>
|
||
</details>
|
||
<div class="desc"><p>JointEmbedCondition(wav, text, length, sample_rate, path, seek_time)</p></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li>builtins.tuple</li>
|
||
</ul>
|
||
<h3>Instance variables</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.JointEmbedCondition.length"><code class="name">var <span class="ident">length</span> : torch.Tensor</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class JointEmbedCondition(tp.NamedTuple):
|
||
wav: torch.Tensor
|
||
text: tp.List[tp.Optional[str]]
|
||
length: torch.Tensor
|
||
sample_rate: tp.List[int]
|
||
path: tp.List[tp.Optional[str]] = []
|
||
seek_time: tp.List[tp.Optional[float]] = []</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Alias for field number 2</p></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.JointEmbedCondition.path"><code class="name">var <span class="ident">path</span> : List[str | None]</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class JointEmbedCondition(tp.NamedTuple):
|
||
wav: torch.Tensor
|
||
text: tp.List[tp.Optional[str]]
|
||
length: torch.Tensor
|
||
sample_rate: tp.List[int]
|
||
path: tp.List[tp.Optional[str]] = []
|
||
seek_time: tp.List[tp.Optional[float]] = []</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Alias for field number 4</p></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.JointEmbedCondition.sample_rate"><code class="name">var <span class="ident">sample_rate</span> : List[int]</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class JointEmbedCondition(tp.NamedTuple):
|
||
wav: torch.Tensor
|
||
text: tp.List[tp.Optional[str]]
|
||
length: torch.Tensor
|
||
sample_rate: tp.List[int]
|
||
path: tp.List[tp.Optional[str]] = []
|
||
seek_time: tp.List[tp.Optional[float]] = []</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Alias for field number 3</p></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.JointEmbedCondition.seek_time"><code class="name">var <span class="ident">seek_time</span> : List[float | None]</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class JointEmbedCondition(tp.NamedTuple):
|
||
wav: torch.Tensor
|
||
text: tp.List[tp.Optional[str]]
|
||
length: torch.Tensor
|
||
sample_rate: tp.List[int]
|
||
path: tp.List[tp.Optional[str]] = []
|
||
seek_time: tp.List[tp.Optional[float]] = []</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Alias for field number 5</p></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.JointEmbedCondition.text"><code class="name">var <span class="ident">text</span> : List[str | None]</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class JointEmbedCondition(tp.NamedTuple):
|
||
wav: torch.Tensor
|
||
text: tp.List[tp.Optional[str]]
|
||
length: torch.Tensor
|
||
sample_rate: tp.List[int]
|
||
path: tp.List[tp.Optional[str]] = []
|
||
seek_time: tp.List[tp.Optional[float]] = []</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Alias for field number 1</p></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.JointEmbedCondition.wav"><code class="name">var <span class="ident">wav</span> : torch.Tensor</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class JointEmbedCondition(tp.NamedTuple):
|
||
wav: torch.Tensor
|
||
text: tp.List[tp.Optional[str]]
|
||
length: torch.Tensor
|
||
sample_rate: tp.List[int]
|
||
path: tp.List[tp.Optional[str]] = []
|
||
seek_time: tp.List[tp.Optional[float]] = []</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Alias for field number 0</p></div>
|
||
</dd>
|
||
</dl>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.JointEmbeddingConditioner"><code class="flex name class">
|
||
<span>class <span class="ident">JointEmbeddingConditioner</span></span>
|
||
<span>(</span><span>dim: int,<br>output_dim: int,<br>device: str,<br>attribute: str,<br>autocast_dtype: str | None = 'float32',<br>quantize: bool = True,<br>n_q: int = 12,<br>bins: int = 1024,<br>**kwargs)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class JointEmbeddingConditioner(BaseConditioner):
|
||
"""Joint embedding conditioning supporting both audio or text conditioning.
|
||
|
||
Args:
|
||
dim (int): Dimension.
|
||
output_dim (int): Output dimension.
|
||
device (str): Device.
|
||
attribute (str): Attribute used by the conditioner.
|
||
autocast_dtype (str): Autocast for the conditioner.
|
||
quantize (bool): Whether to quantize the CLAP embedding.
|
||
n_q (int): Number of residual quantizers (used if quantize is true).
|
||
bins (int): Quantizers' codebooks size (used if quantize is true).
|
||
kwargs: Additional parameters for residual vector quantizer.
|
||
"""
|
||
def __init__(self, dim: int, output_dim: int, device: str, attribute: str,
|
||
autocast_dtype: tp.Optional[str] = 'float32', quantize: bool = True,
|
||
n_q: int = 12, bins: int = 1024, **kwargs):
|
||
super().__init__(dim=dim, output_dim=output_dim)
|
||
self.device = device
|
||
self.attribute = attribute
|
||
if autocast_dtype is None or device == 'cpu':
|
||
self.autocast = TorchAutocast(enabled=False)
|
||
logger.warning("JointEmbeddingConditioner has no autocast, this might lead to NaN.")
|
||
else:
|
||
dtype = getattr(torch, autocast_dtype)
|
||
assert isinstance(dtype, torch.dtype)
|
||
logger.info(f"JointEmbeddingConditioner will be evaluated with autocast as {autocast_dtype}.")
|
||
self.autocast = TorchAutocast(enabled=True, device_type=self.device, dtype=dtype)
|
||
# residual vector quantizer to discretize the conditioned embedding
|
||
self.quantizer: tp.Optional[ResidualVectorQuantizer] = None
|
||
if quantize:
|
||
self.quantizer = ResidualVectorQuantizer(dim, n_q=n_q, bins=bins, **kwargs)
|
||
|
||
def _get_embed(self, x: JointEmbedCondition) -> tp.Tuple[torch.Tensor, torch.Tensor]:
|
||
"""Get joint embedding in latent space from the inputs.
|
||
|
||
Returns:
|
||
tuple[torch.Tensor, torch.Tensor]: Tensor for the latent embedding
|
||
and corresponding empty indexes.
|
||
"""
|
||
raise NotImplementedError()
|
||
|
||
def forward(self, x: JointEmbedCondition) -> ConditionType:
|
||
with self.autocast:
|
||
embed, empty_idx = self._get_embed(x)
|
||
if self.quantizer is not None:
|
||
embed = embed.view(-1, self.dim, 1)
|
||
q_res = self.quantizer(embed, frame_rate=1)
|
||
out_embed = q_res.x.view(-1, self.dim)
|
||
else:
|
||
out_embed = embed
|
||
out_embed = self.output_proj(out_embed).view(-1, 1, self.output_dim)
|
||
mask = torch.ones(*out_embed.shape[:2], device=out_embed.device)
|
||
mask[empty_idx, :] = 0 # zero-out index where the input is non-existant
|
||
out_embed = (out_embed * mask.unsqueeze(-1))
|
||
return out_embed, mask
|
||
|
||
def tokenize(self, x: JointEmbedCondition) -> JointEmbedCondition:
|
||
return x</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Joint embedding conditioning supporting both audio or text conditioning.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>dim</code></strong> : <code>int</code></dt>
|
||
<dd>Dimension.</dd>
|
||
<dt><strong><code>output_dim</code></strong> : <code>int</code></dt>
|
||
<dd>Output dimension.</dd>
|
||
<dt><strong><code>device</code></strong> : <code>str</code></dt>
|
||
<dd>Device.</dd>
|
||
<dt><strong><code>attribute</code></strong> : <code>str</code></dt>
|
||
<dd>Attribute used by the conditioner.</dd>
|
||
<dt><strong><code>autocast_dtype</code></strong> : <code>str</code></dt>
|
||
<dd>Autocast for the conditioner.</dd>
|
||
<dt><strong><code>quantize</code></strong> : <code>bool</code></dt>
|
||
<dd>Whether to quantize the CLAP embedding.</dd>
|
||
<dt><strong><code>n_q</code></strong> : <code>int</code></dt>
|
||
<dd>Number of residual quantizers (used if quantize is true).</dd>
|
||
<dt><strong><code>bins</code></strong> : <code>int</code></dt>
|
||
<dd>Quantizers' codebooks size (used if quantize is true).</dd>
|
||
<dt><strong><code>kwargs</code></strong></dt>
|
||
<dd>Additional parameters for residual vector quantizer.</dd>
|
||
</dl>
|
||
<p>Initializes internal Module state, shared by both nn.Module and ScriptModule.</p></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.conditioners.BaseConditioner" href="#audiocraft.modules.conditioners.BaseConditioner">BaseConditioner</a></li>
|
||
<li>torch.nn.modules.module.Module</li>
|
||
</ul>
|
||
<h3>Subclasses</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.conditioners.CLAPEmbeddingConditioner" href="#audiocraft.modules.conditioners.CLAPEmbeddingConditioner">CLAPEmbeddingConditioner</a></li>
|
||
</ul>
|
||
<h3>Class variables</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.JointEmbeddingConditioner.call_super_init"><code class="name">var <span class="ident">call_super_init</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.JointEmbeddingConditioner.dump_patches"><code class="name">var <span class="ident">dump_patches</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.JointEmbeddingConditioner.training"><code class="name">var <span class="ident">training</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
<h3>Inherited members</h3>
|
||
<ul class="hlist">
|
||
<li><code><b><a title="audiocraft.modules.conditioners.BaseConditioner" href="#audiocraft.modules.conditioners.BaseConditioner">BaseConditioner</a></b></code>:
|
||
<ul class="hlist">
|
||
<li><code><a title="audiocraft.modules.conditioners.BaseConditioner.forward" href="#audiocraft.modules.conditioners.BaseConditioner.forward">forward</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.BaseConditioner.tokenize" href="#audiocraft.modules.conditioners.BaseConditioner.tokenize">tokenize</a></code></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.LUTConditioner"><code class="flex name class">
|
||
<span>class <span class="ident">LUTConditioner</span></span>
|
||
<span>(</span><span>n_bins: int, dim: int, output_dim: int, tokenizer: str, pad_idx: int = 0)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class LUTConditioner(TextConditioner):
|
||
"""Lookup table TextConditioner.
|
||
|
||
Args:
|
||
n_bins (int): Number of bins.
|
||
dim (int): Hidden dim of the model (text-encoder/LUT).
|
||
output_dim (int): Output dim of the conditioner.
|
||
tokenizer (str): Name of the tokenizer.
|
||
pad_idx (int, optional): Index for padding token. Defaults to 0.
|
||
"""
|
||
def __init__(self, n_bins: int, dim: int, output_dim: int, tokenizer: str, pad_idx: int = 0):
|
||
super().__init__(dim, output_dim)
|
||
self.embed = nn.Embedding(n_bins, dim)
|
||
self.tokenizer: Tokenizer
|
||
if tokenizer == 'whitespace':
|
||
self.tokenizer = WhiteSpaceTokenizer(n_bins, pad_idx=pad_idx)
|
||
elif tokenizer == 'noop':
|
||
self.tokenizer = NoopTokenizer(n_bins, pad_idx=pad_idx)
|
||
else:
|
||
raise ValueError(f"unrecognized tokenizer `{tokenizer}`.")
|
||
|
||
def tokenize(self, x: tp.List[tp.Optional[str]]) -> tp.Tuple[torch.Tensor, torch.Tensor]:
|
||
device = self.embed.weight.device
|
||
tokens, mask = self.tokenizer(x)
|
||
tokens, mask = tokens.to(device), mask.to(device)
|
||
return tokens, mask
|
||
|
||
def forward(self, inputs: tp.Tuple[torch.Tensor, torch.Tensor]) -> ConditionType:
|
||
tokens, mask = inputs
|
||
embeds = self.embed(tokens)
|
||
embeds = self.output_proj(embeds)
|
||
embeds = (embeds * mask.unsqueeze(-1))
|
||
return embeds, mask</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Lookup table TextConditioner.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>n_bins</code></strong> : <code>int</code></dt>
|
||
<dd>Number of bins.</dd>
|
||
<dt><strong><code>dim</code></strong> : <code>int</code></dt>
|
||
<dd>Hidden dim of the model (text-encoder/LUT).</dd>
|
||
<dt><strong><code>output_dim</code></strong> : <code>int</code></dt>
|
||
<dd>Output dim of the conditioner.</dd>
|
||
<dt><strong><code>tokenizer</code></strong> : <code>str</code></dt>
|
||
<dd>Name of the tokenizer.</dd>
|
||
<dt><strong><code>pad_idx</code></strong> : <code>int</code>, optional</dt>
|
||
<dd>Index for padding token. Defaults to 0.</dd>
|
||
</dl>
|
||
<p>Initializes internal Module state, shared by both nn.Module and ScriptModule.</p></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.conditioners.TextConditioner" href="#audiocraft.modules.conditioners.TextConditioner">TextConditioner</a></li>
|
||
<li><a title="audiocraft.modules.conditioners.BaseConditioner" href="#audiocraft.modules.conditioners.BaseConditioner">BaseConditioner</a></li>
|
||
<li>torch.nn.modules.module.Module</li>
|
||
</ul>
|
||
<h3>Class variables</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.LUTConditioner.call_super_init"><code class="name">var <span class="ident">call_super_init</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.LUTConditioner.dump_patches"><code class="name">var <span class="ident">dump_patches</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.LUTConditioner.training"><code class="name">var <span class="ident">training</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
<h3>Inherited members</h3>
|
||
<ul class="hlist">
|
||
<li><code><b><a title="audiocraft.modules.conditioners.TextConditioner" href="#audiocraft.modules.conditioners.TextConditioner">TextConditioner</a></b></code>:
|
||
<ul class="hlist">
|
||
<li><code><a title="audiocraft.modules.conditioners.TextConditioner.forward" href="#audiocraft.modules.conditioners.BaseConditioner.forward">forward</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.TextConditioner.tokenize" href="#audiocraft.modules.conditioners.BaseConditioner.tokenize">tokenize</a></code></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.NoopTokenizer"><code class="flex name class">
|
||
<span>class <span class="ident">NoopTokenizer</span></span>
|
||
<span>(</span><span>n_bins: int, pad_idx: int = 0)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class NoopTokenizer(Tokenizer):
|
||
"""This tokenizer should be used for global conditioners such as: artist, genre, key, etc.
|
||
The difference between this and WhiteSpaceTokenizer is that NoopTokenizer does not split
|
||
strings, so "Jeff Buckley" will get it's own index. Whereas WhiteSpaceTokenizer will
|
||
split it to ["Jeff", "Buckley"] and return an index per word.
|
||
|
||
For example:
|
||
["Queen", "ABBA", "Jeff Buckley"] => [43, 55, 101]
|
||
["Metal", "Rock", "Classical"] => [0, 223, 51]
|
||
"""
|
||
def __init__(self, n_bins: int, pad_idx: int = 0):
|
||
self.n_bins = n_bins
|
||
self.pad_idx = pad_idx
|
||
|
||
def __call__(self, texts: tp.List[tp.Optional[str]]) -> tp.Tuple[torch.Tensor, torch.Tensor]:
|
||
output, lengths = [], []
|
||
for text in texts:
|
||
# if current sample doesn't have a certain attribute, replace with pad token
|
||
if text is None:
|
||
output.append(self.pad_idx)
|
||
lengths.append(0)
|
||
else:
|
||
output.append(hash_trick(text, self.n_bins))
|
||
lengths.append(1)
|
||
|
||
tokens = torch.LongTensor(output).unsqueeze(1)
|
||
mask = length_to_mask(torch.IntTensor(lengths)).int()
|
||
return tokens, mask</code></pre>
|
||
</details>
|
||
<div class="desc"><p>This tokenizer should be used for global conditioners such as: artist, genre, key, etc.
|
||
The difference between this and WhiteSpaceTokenizer is that NoopTokenizer does not split
|
||
strings, so "Jeff Buckley" will get it's own index. Whereas WhiteSpaceTokenizer will
|
||
split it to ["Jeff", "Buckley"] and return an index per word.</p>
|
||
<p>For example:
|
||
["Queen", "ABBA", "Jeff Buckley"] => [43, 55, 101]
|
||
["Metal", "Rock", "Classical"] => [0, 223, 51]</p></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.conditioners.Tokenizer" href="#audiocraft.modules.conditioners.Tokenizer">Tokenizer</a></li>
|
||
</ul>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.SegmentWithAttributes"><code class="flex name class">
|
||
<span>class <span class="ident">SegmentWithAttributes</span></span>
|
||
<span>(</span><span>meta: <a title="audiocraft.data.audio_dataset.AudioMeta" href="../data/audio_dataset.html#audiocraft.data.audio_dataset.AudioMeta">AudioMeta</a>,<br>seek_time: float,<br>n_frames: int,<br>total_frames: int,<br>sample_rate: int,<br>channels: int)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class SegmentWithAttributes(SegmentInfo):
|
||
"""Base class for all dataclasses that are used for conditioning.
|
||
All child classes should implement `to_condition_attributes` that converts
|
||
the existing attributes to a dataclass of type ConditioningAttributes.
|
||
"""
|
||
def to_condition_attributes(self) -> ConditioningAttributes:
|
||
raise NotImplementedError()</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Base class for all dataclasses that are used for conditioning.
|
||
All child classes should implement <code>to_condition_attributes</code> that converts
|
||
the existing attributes to a dataclass of type ConditioningAttributes.</p></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.data.audio_dataset.SegmentInfo" href="../data/audio_dataset.html#audiocraft.data.audio_dataset.SegmentInfo">SegmentInfo</a></li>
|
||
<li><a title="audiocraft.data.audio_dataset.BaseInfo" href="../data/audio_dataset.html#audiocraft.data.audio_dataset.BaseInfo">BaseInfo</a></li>
|
||
</ul>
|
||
<h3>Subclasses</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.data.info_audio_dataset.AudioInfo" href="../data/info_audio_dataset.html#audiocraft.data.info_audio_dataset.AudioInfo">AudioInfo</a></li>
|
||
<li><a title="audiocraft.data.sound_dataset.SoundInfo" href="../data/sound_dataset.html#audiocraft.data.sound_dataset.SoundInfo">SoundInfo</a></li>
|
||
</ul>
|
||
<h3>Class variables</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.SegmentWithAttributes.channels"><code class="name">var <span class="ident">channels</span> : int</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.SegmentWithAttributes.meta"><code class="name">var <span class="ident">meta</span> : <a title="audiocraft.data.audio_dataset.AudioMeta" href="../data/audio_dataset.html#audiocraft.data.audio_dataset.AudioMeta">AudioMeta</a></code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.SegmentWithAttributes.n_frames"><code class="name">var <span class="ident">n_frames</span> : int</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.SegmentWithAttributes.sample_rate"><code class="name">var <span class="ident">sample_rate</span> : int</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.SegmentWithAttributes.seek_time"><code class="name">var <span class="ident">seek_time</span> : float</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.SegmentWithAttributes.total_frames"><code class="name">var <span class="ident">total_frames</span> : int</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
<h3>Methods</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.SegmentWithAttributes.to_condition_attributes"><code class="name flex">
|
||
<span>def <span class="ident">to_condition_attributes</span></span>(<span>self) ‑> <a title="audiocraft.modules.conditioners.ConditioningAttributes" href="#audiocraft.modules.conditioners.ConditioningAttributes">ConditioningAttributes</a></span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def to_condition_attributes(self) -> ConditioningAttributes:
|
||
raise NotImplementedError()</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.StyleConditioner"><code class="flex name class">
|
||
<span>class <span class="ident">StyleConditioner</span></span>
|
||
<span>(</span><span>transformer_scale: str = 'default',<br>ds_factor: int = 15,<br>encodec_n_q: int = 4,<br>n_q_out: int = 6,<br>eval_q: int = 3,<br>q_dropout: bool = True,<br>bins: int = 1024,<br>varying_lengths: List[float] = [1.5, 4.5],<br>batch_norm: bool = True,<br>rvq_threshold_ema_dead_code: float = 0.1,<br>**kwargs)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class StyleConditioner(FeatureExtractor):
|
||
"""Conditioner from the paper AUDIO CONDITIONING FOR MUSIC GENERATION VIA
|
||
DISCRETE BOTTLENECK FEATURES.
|
||
Given an audio input, it is passed through a Feature Extractor and a
|
||
transformer encoder. Then it is quantized through RVQ.
|
||
|
||
Args:
|
||
transformer_scale (str): size of the transformer. See in the __init__ to have more infos.
|
||
ds_factor (int): the downsampling factor applied to the representation after quantization.
|
||
encodec_n_q (int): if encodec is used as a feature extractor it sets the number of
|
||
quantization streams used in it.
|
||
n_q_out (int): the number of quantization streams used for the RVQ. If increased, there
|
||
is more information passing as a conditioning.
|
||
eval_q (int): the number of quantization streams used for the RVQ at evaluation time.
|
||
q_dropout (bool): if True, at training time, a random number of stream is sampled
|
||
at each step in the interval [1, n_q_out].
|
||
bins (int): the codebook size used for each quantization stream.
|
||
varying_lengths (List[float]): list of the min and max duration in seconds for the
|
||
randomly subsampled excerpt at training time. For each step a length is sampled
|
||
in this interval.
|
||
batch_norm (bool): use of batch normalization after the transformer. Stabilizes the
|
||
training.
|
||
rvq_threshold_ema_dead_code (float): threshold for dropping dead codes in the
|
||
RVQ.
|
||
"""
|
||
def __init__(self, transformer_scale: str = 'default', ds_factor: int = 15, encodec_n_q: int = 4,
|
||
n_q_out: int = 6, eval_q: int = 3, q_dropout: bool = True, bins: int = 1024,
|
||
varying_lengths: tp.List[float] = [1.5, 4.5],
|
||
batch_norm: bool = True, rvq_threshold_ema_dead_code: float = 0.1,
|
||
**kwargs):
|
||
tr_args: tp.Dict[str, tp.Any]
|
||
if transformer_scale == 'xsmall':
|
||
tr_args = {'d_model': 256, 'num_heads': 8, 'num_layers': 4}
|
||
elif transformer_scale == 'large':
|
||
tr_args = {'d_model': 1024, 'num_heads': 16, 'num_layers': 24}
|
||
elif transformer_scale == 'default':
|
||
tr_args = {'d_model': 512, 'num_heads': 8, 'num_layers': 8}
|
||
elif transformer_scale == 'none':
|
||
tr_args = {'d_model': 512}
|
||
tr_args.update({
|
||
'memory_efficient': True, 'activation': 'gelu',
|
||
'norm_first': True, 'causal': False, 'layer_scale': None,
|
||
'bias_ff': False, 'bias_attn': False,
|
||
})
|
||
dim = tr_args['d_model']
|
||
super().__init__(dim=dim, encodec_n_q=encodec_n_q, **kwargs)
|
||
|
||
self.ds_factor = ds_factor
|
||
if transformer_scale == 'none':
|
||
self.transformer = None
|
||
else:
|
||
self.transformer = StreamingTransformer(dim_feedforward=int(4 * dim), **tr_args)
|
||
self.n_q_out = n_q_out
|
||
self.eval_q = eval_q
|
||
self.rvq = None
|
||
if n_q_out > 0:
|
||
self.rvq = ResidualVectorQuantizer(dim, n_q=n_q_out, q_dropout=q_dropout, bins=bins,
|
||
threshold_ema_dead_code=rvq_threshold_ema_dead_code)
|
||
self.autocast = TorchAutocast(enabled=self.device != 'cpu', device_type=self.device, dtype=torch.float32)
|
||
self.varying_lengths = varying_lengths
|
||
self.batch_norm = None
|
||
if batch_norm:
|
||
self.batch_norm = nn.BatchNorm1d(dim, affine=False)
|
||
self.mask = None
|
||
|
||
def _get_wav_embedding(self, wav: WavCondition) -> torch.Tensor:
|
||
with self.autocast:
|
||
# Sample the length of the excerpts
|
||
if self.varying_lengths and self.training:
|
||
assert len(self.varying_lengths) == 2
|
||
length = random.uniform(self.varying_lengths[0], self.varying_lengths[1])
|
||
self.length_subwav = int(length * self.sample_rate)
|
||
z1 = super()._get_wav_embedding(wav)
|
||
if self.compute_mask:
|
||
self.mask = self.temp_mask # type: ignore
|
||
self.temp_mask = None
|
||
|
||
if self.transformer is not None:
|
||
out1 = self.transformer(z1)
|
||
else:
|
||
out1 = z1
|
||
if self.batch_norm:
|
||
out1 = self.batch_norm(out1.transpose(1, 2)).transpose(1, 2)
|
||
# Apply quantization
|
||
if self.rvq:
|
||
if self.training:
|
||
self.rvq.set_num_codebooks(self.n_q_out)
|
||
else:
|
||
self.rvq.set_num_codebooks(self.eval_q)
|
||
out1 = self.rvq(out1.transpose(1, 2), frame_rate=1.)
|
||
if self.training:
|
||
flashy.distrib.average_tensors(self.rvq.buffers())
|
||
out1 = out1.x.transpose(1, 2)
|
||
# Apply fix downsample
|
||
out1 = out1[:, ::self.ds_factor]
|
||
|
||
return out1
|
||
|
||
def set_params(self, eval_q: int = 3,
|
||
excerpt_length: float = 3.0,
|
||
ds_factor: tp.Optional[int] = None, encodec_n_q: tp.Optional[int] = None):
|
||
"""Modify the parameters of the SSL or introduce new parameters to add noise to
|
||
the conditioning or to downsample it
|
||
|
||
Args:
|
||
eval_q (int): number of codebooks used when evaluating the model
|
||
excerpt_length (float): the length of the excerpts used to condition the model
|
||
"""
|
||
self.eval_q = eval_q
|
||
self.length_subwav = int(excerpt_length * self.sample_rate)
|
||
if ds_factor is not None:
|
||
self.ds_factor = ds_factor
|
||
if encodec_n_q is not None:
|
||
self.encodec_n_q = encodec_n_q
|
||
|
||
def _downsampling_factor(self):
|
||
df = super()._downsampling_factor()
|
||
return df * self.ds_factor
|
||
|
||
def forward(self, x: WavCondition) -> ConditionType:
|
||
wav, lengths, *_ = x
|
||
|
||
embeds = self._get_wav_embedding(x)
|
||
embeds = embeds.to(self.output_proj.weight)
|
||
embeds = self.output_proj(embeds)
|
||
|
||
lengths = lengths / self._downsampling_factor()
|
||
mask = length_to_mask(lengths, max_len=embeds.shape[1]).int() # type: ignore
|
||
|
||
embeds = (embeds * mask.unsqueeze(2).to(self.device))
|
||
|
||
return embeds, mask</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Conditioner from the paper AUDIO CONDITIONING FOR MUSIC GENERATION VIA
|
||
DISCRETE BOTTLENECK FEATURES.
|
||
Given an audio input, it is passed through a Feature Extractor and a
|
||
transformer encoder. Then it is quantized through RVQ.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>transformer_scale</code></strong> : <code>str</code></dt>
|
||
<dd>size of the transformer. See in the <strong>init</strong> to have more infos.</dd>
|
||
<dt><strong><code>ds_factor</code></strong> : <code>int</code></dt>
|
||
<dd>the downsampling factor applied to the representation after quantization.</dd>
|
||
<dt><strong><code>encodec_n_q</code></strong> : <code>int</code></dt>
|
||
<dd>if encodec is used as a feature extractor it sets the number of
|
||
quantization streams used in it.</dd>
|
||
<dt><strong><code>n_q_out</code></strong> : <code>int</code></dt>
|
||
<dd>the number of quantization streams used for the RVQ. If increased, there
|
||
is more information passing as a conditioning.</dd>
|
||
<dt><strong><code>eval_q</code></strong> : <code>int</code></dt>
|
||
<dd>the number of quantization streams used for the RVQ at evaluation time.</dd>
|
||
<dt><strong><code>q_dropout</code></strong> : <code>bool</code></dt>
|
||
<dd>if True, at training time, a random number of stream is sampled
|
||
at each step in the interval [1, n_q_out].</dd>
|
||
<dt><strong><code>bins</code></strong> : <code>int</code></dt>
|
||
<dd>the codebook size used for each quantization stream.</dd>
|
||
<dt><strong><code>varying_lengths</code></strong> : <code>List[float]</code></dt>
|
||
<dd>list of the min and max duration in seconds for the
|
||
randomly subsampled excerpt at training time. For each step a length is sampled
|
||
in this interval.</dd>
|
||
<dt><strong><code>batch_norm</code></strong> : <code>bool</code></dt>
|
||
<dd>use of batch normalization after the transformer. Stabilizes the
|
||
training.</dd>
|
||
<dt><strong><code>rvq_threshold_ema_dead_code</code></strong> : <code>float</code></dt>
|
||
<dd>threshold for dropping dead codes in the
|
||
RVQ.</dd>
|
||
</dl>
|
||
<p>Initializes internal Module state, shared by both nn.Module and ScriptModule.</p></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.conditioners.FeatureExtractor" href="#audiocraft.modules.conditioners.FeatureExtractor">FeatureExtractor</a></li>
|
||
<li><a title="audiocraft.modules.conditioners.WaveformConditioner" href="#audiocraft.modules.conditioners.WaveformConditioner">WaveformConditioner</a></li>
|
||
<li><a title="audiocraft.modules.conditioners.BaseConditioner" href="#audiocraft.modules.conditioners.BaseConditioner">BaseConditioner</a></li>
|
||
<li>torch.nn.modules.module.Module</li>
|
||
</ul>
|
||
<h3>Class variables</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.StyleConditioner.call_super_init"><code class="name">var <span class="ident">call_super_init</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.StyleConditioner.dump_patches"><code class="name">var <span class="ident">dump_patches</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.StyleConditioner.training"><code class="name">var <span class="ident">training</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
<h3>Methods</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.StyleConditioner.set_params"><code class="name flex">
|
||
<span>def <span class="ident">set_params</span></span>(<span>self,<br>eval_q: int = 3,<br>excerpt_length: float = 3.0,<br>ds_factor: int | None = None,<br>encodec_n_q: int | None = None)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def set_params(self, eval_q: int = 3,
|
||
excerpt_length: float = 3.0,
|
||
ds_factor: tp.Optional[int] = None, encodec_n_q: tp.Optional[int] = None):
|
||
"""Modify the parameters of the SSL or introduce new parameters to add noise to
|
||
the conditioning or to downsample it
|
||
|
||
Args:
|
||
eval_q (int): number of codebooks used when evaluating the model
|
||
excerpt_length (float): the length of the excerpts used to condition the model
|
||
"""
|
||
self.eval_q = eval_q
|
||
self.length_subwav = int(excerpt_length * self.sample_rate)
|
||
if ds_factor is not None:
|
||
self.ds_factor = ds_factor
|
||
if encodec_n_q is not None:
|
||
self.encodec_n_q = encodec_n_q</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Modify the parameters of the SSL or introduce new parameters to add noise to
|
||
the conditioning or to downsample it</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>eval_q</code></strong> : <code>int</code></dt>
|
||
<dd>number of codebooks used when evaluating the model</dd>
|
||
<dt><strong><code>excerpt_length</code></strong> : <code>float</code></dt>
|
||
<dd>the length of the excerpts used to condition the model</dd>
|
||
</dl></div>
|
||
</dd>
|
||
</dl>
|
||
<h3>Inherited members</h3>
|
||
<ul class="hlist">
|
||
<li><code><b><a title="audiocraft.modules.conditioners.FeatureExtractor" href="#audiocraft.modules.conditioners.FeatureExtractor">FeatureExtractor</a></b></code>:
|
||
<ul class="hlist">
|
||
<li><code><a title="audiocraft.modules.conditioners.FeatureExtractor.forward" href="#audiocraft.modules.conditioners.WaveformConditioner.forward">forward</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.FeatureExtractor.tokenize" href="#audiocraft.modules.conditioners.BaseConditioner.tokenize">tokenize</a></code></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.SymbolicCondition"><code class="flex name class">
|
||
<span>class <span class="ident">SymbolicCondition</span></span>
|
||
<span>(</span><span>frame_chords: torch.Tensor | None = None, melody: torch.Tensor | None = None)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class SymbolicCondition(tp.NamedTuple):
|
||
frame_chords: tp.Optional[torch.Tensor] = None
|
||
melody: tp.Optional[torch.Tensor] = None</code></pre>
|
||
</details>
|
||
<div class="desc"><p>SymbolicCondition(frame_chords, melody)</p></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li>builtins.tuple</li>
|
||
</ul>
|
||
<h3>Instance variables</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.SymbolicCondition.frame_chords"><code class="name">var <span class="ident">frame_chords</span> : torch.Tensor | None</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class SymbolicCondition(tp.NamedTuple):
|
||
frame_chords: tp.Optional[torch.Tensor] = None
|
||
melody: tp.Optional[torch.Tensor] = None</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Alias for field number 0</p></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.SymbolicCondition.melody"><code class="name">var <span class="ident">melody</span> : torch.Tensor | None</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class SymbolicCondition(tp.NamedTuple):
|
||
frame_chords: tp.Optional[torch.Tensor] = None
|
||
melody: tp.Optional[torch.Tensor] = None</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Alias for field number 1</p></div>
|
||
</dd>
|
||
</dl>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.T5Conditioner"><code class="flex name class">
|
||
<span>class <span class="ident">T5Conditioner</span></span>
|
||
<span>(</span><span>name: str,<br>output_dim: int,<br>finetune: bool,<br>device: str,<br>autocast_dtype: str | None = 'float32',<br>word_dropout: float = 0.0,<br>normalize_text: bool = False)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class T5Conditioner(TextConditioner):
|
||
"""T5-based TextConditioner.
|
||
|
||
Args:
|
||
name (str): Name of the T5 model.
|
||
output_dim (int): Output dim of the conditioner.
|
||
finetune (bool): Whether to fine-tune T5 at train time.
|
||
device (str): Device for T5 Conditioner.
|
||
autocast_dtype (tp.Optional[str], optional): Autocast dtype.
|
||
word_dropout (float, optional): Word dropout probability.
|
||
normalize_text (bool, optional): Whether to apply text normalization.
|
||
"""
|
||
MODELS = ["t5-small", "t5-base", "t5-large", "t5-3b", "t5-11b",
|
||
"google/flan-t5-small", "google/flan-t5-base", "google/flan-t5-large",
|
||
"google/flan-t5-xl", "google/flan-t5-xxl"]
|
||
MODELS_DIMS = {
|
||
"t5-small": 512,
|
||
"t5-base": 768,
|
||
"t5-large": 1024,
|
||
"t5-3b": 1024,
|
||
"t5-11b": 1024,
|
||
"google/flan-t5-small": 512,
|
||
"google/flan-t5-base": 768,
|
||
"google/flan-t5-large": 1024,
|
||
"google/flan-t5-3b": 1024,
|
||
"google/flan-t5-11b": 1024,
|
||
}
|
||
|
||
def __init__(self, name: str, output_dim: int, finetune: bool, device: str,
|
||
autocast_dtype: tp.Optional[str] = 'float32', word_dropout: float = 0.,
|
||
normalize_text: bool = False):
|
||
assert name in self.MODELS, f"Unrecognized t5 model name (should in {self.MODELS})"
|
||
super().__init__(self.MODELS_DIMS[name], output_dim)
|
||
self.device = device
|
||
self.name = name
|
||
self.finetune = finetune
|
||
self.word_dropout = word_dropout
|
||
if autocast_dtype is None or self.device == 'cpu':
|
||
self.autocast = TorchAutocast(enabled=False)
|
||
if self.device != 'cpu':
|
||
logger.warning("T5 has no autocast, this might lead to NaN")
|
||
else:
|
||
dtype = getattr(torch, autocast_dtype)
|
||
assert isinstance(dtype, torch.dtype)
|
||
logger.info(f"T5 will be evaluated with autocast as {autocast_dtype}")
|
||
self.autocast = TorchAutocast(enabled=True, device_type=self.device, dtype=dtype)
|
||
# Let's disable logging temporarily because T5 will vomit some errors otherwise.
|
||
# thanks https://gist.github.com/simon-weber/7853144
|
||
previous_level = logging.root.manager.disable
|
||
logging.disable(logging.ERROR)
|
||
with warnings.catch_warnings():
|
||
warnings.simplefilter("ignore")
|
||
try:
|
||
self.t5_tokenizer = T5Tokenizer.from_pretrained(name)
|
||
t5 = T5EncoderModel.from_pretrained(name).train(mode=finetune)
|
||
finally:
|
||
logging.disable(previous_level)
|
||
if finetune:
|
||
self.t5 = t5
|
||
else:
|
||
# this makes sure that the t5 models is not part
|
||
# of the saved checkpoint
|
||
self.__dict__['t5'] = t5.to(device)
|
||
|
||
self.normalize_text = normalize_text
|
||
if normalize_text:
|
||
self.text_normalizer = WhiteSpaceTokenizer(1, lemma=True, stopwords=True)
|
||
|
||
def tokenize(self, x: tp.List[tp.Optional[str]]) -> tp.Dict[str, torch.Tensor]:
|
||
# if current sample doesn't have a certain attribute, replace with empty string
|
||
entries: tp.List[str] = [xi if xi is not None else "" for xi in x]
|
||
if self.normalize_text:
|
||
_, _, entries = self.text_normalizer(entries, return_text=True)
|
||
if self.word_dropout > 0. and self.training:
|
||
new_entries = []
|
||
for entry in entries:
|
||
words = [word for word in entry.split(" ") if random.random() >= self.word_dropout]
|
||
new_entries.append(" ".join(words))
|
||
entries = new_entries
|
||
|
||
empty_idx = torch.LongTensor([i for i, xi in enumerate(entries) if xi == ""])
|
||
|
||
inputs = self.t5_tokenizer(entries, return_tensors='pt', padding=True).to(self.device)
|
||
mask = inputs['attention_mask']
|
||
mask[empty_idx, :] = 0 # zero-out index where the input is non-existant
|
||
return inputs
|
||
|
||
def forward(self, inputs: tp.Dict[str, torch.Tensor]) -> ConditionType:
|
||
mask = inputs['attention_mask']
|
||
with torch.set_grad_enabled(self.finetune), self.autocast:
|
||
embeds = self.t5(**inputs).last_hidden_state
|
||
embeds = self.output_proj(embeds.to(self.output_proj.weight))
|
||
embeds = (embeds * mask.unsqueeze(-1))
|
||
return embeds, mask</code></pre>
|
||
</details>
|
||
<div class="desc"><p>T5-based TextConditioner.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>name</code></strong> : <code>str</code></dt>
|
||
<dd>Name of the T5 model.</dd>
|
||
<dt><strong><code>output_dim</code></strong> : <code>int</code></dt>
|
||
<dd>Output dim of the conditioner.</dd>
|
||
<dt><strong><code>finetune</code></strong> : <code>bool</code></dt>
|
||
<dd>Whether to fine-tune T5 at train time.</dd>
|
||
<dt><strong><code>device</code></strong> : <code>str</code></dt>
|
||
<dd>Device for T5 Conditioner.</dd>
|
||
<dt><strong><code>autocast_dtype</code></strong> : <code>tp.Optional[str]</code>, optional</dt>
|
||
<dd>Autocast dtype.</dd>
|
||
<dt><strong><code>word_dropout</code></strong> : <code>float</code>, optional</dt>
|
||
<dd>Word dropout probability.</dd>
|
||
<dt><strong><code>normalize_text</code></strong> : <code>bool</code>, optional</dt>
|
||
<dd>Whether to apply text normalization.</dd>
|
||
</dl>
|
||
<p>Initializes internal Module state, shared by both nn.Module and ScriptModule.</p></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.conditioners.TextConditioner" href="#audiocraft.modules.conditioners.TextConditioner">TextConditioner</a></li>
|
||
<li><a title="audiocraft.modules.conditioners.BaseConditioner" href="#audiocraft.modules.conditioners.BaseConditioner">BaseConditioner</a></li>
|
||
<li>torch.nn.modules.module.Module</li>
|
||
</ul>
|
||
<h3>Class variables</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.T5Conditioner.MODELS"><code class="name">var <span class="ident">MODELS</span></code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.T5Conditioner.MODELS_DIMS"><code class="name">var <span class="ident">MODELS_DIMS</span></code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.T5Conditioner.call_super_init"><code class="name">var <span class="ident">call_super_init</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.T5Conditioner.dump_patches"><code class="name">var <span class="ident">dump_patches</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.T5Conditioner.training"><code class="name">var <span class="ident">training</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
<h3>Inherited members</h3>
|
||
<ul class="hlist">
|
||
<li><code><b><a title="audiocraft.modules.conditioners.TextConditioner" href="#audiocraft.modules.conditioners.TextConditioner">TextConditioner</a></b></code>:
|
||
<ul class="hlist">
|
||
<li><code><a title="audiocraft.modules.conditioners.TextConditioner.forward" href="#audiocraft.modules.conditioners.BaseConditioner.forward">forward</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.TextConditioner.tokenize" href="#audiocraft.modules.conditioners.BaseConditioner.tokenize">tokenize</a></code></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.TextConditioner"><code class="flex name class">
|
||
<span>class <span class="ident">TextConditioner</span></span>
|
||
<span>(</span><span>dim: int, output_dim: int)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class TextConditioner(BaseConditioner):
|
||
...</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Base model for all conditioner modules.
|
||
We allow the output dim to be different than the hidden dim for two reasons:
|
||
1) keep our LUTs small when the vocab is large;
|
||
2) make all condition dims consistent.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>dim</code></strong> : <code>int</code></dt>
|
||
<dd>Hidden dim of the model.</dd>
|
||
<dt><strong><code>output_dim</code></strong> : <code>int</code></dt>
|
||
<dd>Output dim of the conditioner.</dd>
|
||
</dl>
|
||
<p>Initializes internal Module state, shared by both nn.Module and ScriptModule.</p></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.conditioners.BaseConditioner" href="#audiocraft.modules.conditioners.BaseConditioner">BaseConditioner</a></li>
|
||
<li>torch.nn.modules.module.Module</li>
|
||
</ul>
|
||
<h3>Subclasses</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.conditioners.LUTConditioner" href="#audiocraft.modules.conditioners.LUTConditioner">LUTConditioner</a></li>
|
||
<li><a title="audiocraft.modules.conditioners.T5Conditioner" href="#audiocraft.modules.conditioners.T5Conditioner">T5Conditioner</a></li>
|
||
</ul>
|
||
<h3>Class variables</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.TextConditioner.call_super_init"><code class="name">var <span class="ident">call_super_init</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.TextConditioner.dump_patches"><code class="name">var <span class="ident">dump_patches</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.TextConditioner.training"><code class="name">var <span class="ident">training</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
<h3>Inherited members</h3>
|
||
<ul class="hlist">
|
||
<li><code><b><a title="audiocraft.modules.conditioners.BaseConditioner" href="#audiocraft.modules.conditioners.BaseConditioner">BaseConditioner</a></b></code>:
|
||
<ul class="hlist">
|
||
<li><code><a title="audiocraft.modules.conditioners.BaseConditioner.forward" href="#audiocraft.modules.conditioners.BaseConditioner.forward">forward</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.BaseConditioner.tokenize" href="#audiocraft.modules.conditioners.BaseConditioner.tokenize">tokenize</a></code></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.Tokenizer"><code class="flex name class">
|
||
<span>class <span class="ident">Tokenizer</span></span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class Tokenizer:
|
||
"""Base tokenizer implementation
|
||
(in case we want to introduce more advances tokenizers in the future).
|
||
"""
|
||
def __call__(self, texts: tp.List[tp.Optional[str]]) -> tp.Tuple[torch.Tensor, torch.Tensor]:
|
||
raise NotImplementedError()</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Base tokenizer implementation
|
||
(in case we want to introduce more advances tokenizers in the future).</p></div>
|
||
<h3>Subclasses</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.conditioners.NoopTokenizer" href="#audiocraft.modules.conditioners.NoopTokenizer">NoopTokenizer</a></li>
|
||
<li><a title="audiocraft.modules.conditioners.WhiteSpaceTokenizer" href="#audiocraft.modules.conditioners.WhiteSpaceTokenizer">WhiteSpaceTokenizer</a></li>
|
||
</ul>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.WavCondition"><code class="flex name class">
|
||
<span>class <span class="ident">WavCondition</span></span>
|
||
<span>(</span><span>wav: torch.Tensor,<br>length: torch.Tensor,<br>sample_rate: List[int],<br>path: List[str | None] = [],<br>seek_time: List[float | None] = [])</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class WavCondition(tp.NamedTuple):
|
||
wav: torch.Tensor
|
||
length: torch.Tensor
|
||
sample_rate: tp.List[int]
|
||
path: tp.List[tp.Optional[str]] = []
|
||
seek_time: tp.List[tp.Optional[float]] = []</code></pre>
|
||
</details>
|
||
<div class="desc"><p>WavCondition(wav, length, sample_rate, path, seek_time)</p></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li>builtins.tuple</li>
|
||
</ul>
|
||
<h3>Instance variables</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.WavCondition.length"><code class="name">var <span class="ident">length</span> : torch.Tensor</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class WavCondition(tp.NamedTuple):
|
||
wav: torch.Tensor
|
||
length: torch.Tensor
|
||
sample_rate: tp.List[int]
|
||
path: tp.List[tp.Optional[str]] = []
|
||
seek_time: tp.List[tp.Optional[float]] = []</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Alias for field number 1</p></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.WavCondition.path"><code class="name">var <span class="ident">path</span> : List[str | None]</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class WavCondition(tp.NamedTuple):
|
||
wav: torch.Tensor
|
||
length: torch.Tensor
|
||
sample_rate: tp.List[int]
|
||
path: tp.List[tp.Optional[str]] = []
|
||
seek_time: tp.List[tp.Optional[float]] = []</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Alias for field number 3</p></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.WavCondition.sample_rate"><code class="name">var <span class="ident">sample_rate</span> : List[int]</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class WavCondition(tp.NamedTuple):
|
||
wav: torch.Tensor
|
||
length: torch.Tensor
|
||
sample_rate: tp.List[int]
|
||
path: tp.List[tp.Optional[str]] = []
|
||
seek_time: tp.List[tp.Optional[float]] = []</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Alias for field number 2</p></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.WavCondition.seek_time"><code class="name">var <span class="ident">seek_time</span> : List[float | None]</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class WavCondition(tp.NamedTuple):
|
||
wav: torch.Tensor
|
||
length: torch.Tensor
|
||
sample_rate: tp.List[int]
|
||
path: tp.List[tp.Optional[str]] = []
|
||
seek_time: tp.List[tp.Optional[float]] = []</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Alias for field number 4</p></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.WavCondition.wav"><code class="name">var <span class="ident">wav</span> : torch.Tensor</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class WavCondition(tp.NamedTuple):
|
||
wav: torch.Tensor
|
||
length: torch.Tensor
|
||
sample_rate: tp.List[int]
|
||
path: tp.List[tp.Optional[str]] = []
|
||
seek_time: tp.List[tp.Optional[float]] = []</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Alias for field number 0</p></div>
|
||
</dd>
|
||
</dl>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.WaveformConditioner"><code class="flex name class">
|
||
<span>class <span class="ident">WaveformConditioner</span></span>
|
||
<span>(</span><span>dim: int, output_dim: int, device: torch.device | str)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class WaveformConditioner(BaseConditioner):
|
||
"""Base class for all conditioners that take a waveform as input.
|
||
Classes that inherit must implement `_get_wav_embedding` that outputs
|
||
a continuous tensor, and `_downsampling_factor` that returns the down-sampling
|
||
factor of the embedding model.
|
||
|
||
Args:
|
||
dim (int): The internal representation dimension.
|
||
output_dim (int): Output dimension.
|
||
device (tp.Union[torch.device, str]): Device.
|
||
"""
|
||
def __init__(self, dim: int, output_dim: int, device: tp.Union[torch.device, str]):
|
||
super().__init__(dim, output_dim)
|
||
self.device = device
|
||
# if False no masking is done, used in ChromaStemConditioner when completing by periodicity a sample.
|
||
self._use_masking = True
|
||
|
||
def tokenize(self, x: WavCondition) -> WavCondition:
|
||
wav, length, sample_rate, path, seek_time = x
|
||
assert length is not None
|
||
return WavCondition(wav.to(self.device), length.to(self.device), sample_rate, path, seek_time)
|
||
|
||
def _get_wav_embedding(self, x: WavCondition) -> torch.Tensor:
|
||
"""Gets as input a WavCondition and returns a dense embedding."""
|
||
raise NotImplementedError()
|
||
|
||
def _downsampling_factor(self):
|
||
"""Returns the downsampling factor of the embedding model."""
|
||
raise NotImplementedError()
|
||
|
||
def forward(self, x: WavCondition) -> ConditionType:
|
||
"""Extract condition embedding and mask from a waveform and its metadata.
|
||
Args:
|
||
x (WavCondition): Waveform condition containing raw waveform and metadata.
|
||
Returns:
|
||
ConditionType: a dense vector representing the conditioning along with its mask
|
||
"""
|
||
wav, lengths, *_ = x
|
||
with torch.no_grad():
|
||
embeds = self._get_wav_embedding(x)
|
||
if hasattr(self, 'output_proj'):
|
||
embeds = embeds.to(self.output_proj.weight)
|
||
embeds = self.output_proj(embeds)
|
||
|
||
if lengths is not None and self._use_masking:
|
||
lengths = lengths / self._downsampling_factor()
|
||
mask = length_to_mask(lengths, max_len=embeds.shape[1]).int() # type: ignore
|
||
else:
|
||
mask = torch.ones_like(embeds[..., 0])
|
||
embeds = (embeds * mask.unsqueeze(-1))
|
||
return embeds, mask</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Base class for all conditioners that take a waveform as input.
|
||
Classes that inherit must implement <code>_get_wav_embedding</code> that outputs
|
||
a continuous tensor, and <code>_downsampling_factor</code> that returns the down-sampling
|
||
factor of the embedding model.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>dim</code></strong> : <code>int</code></dt>
|
||
<dd>The internal representation dimension.</dd>
|
||
<dt><strong><code>output_dim</code></strong> : <code>int</code></dt>
|
||
<dd>Output dimension.</dd>
|
||
<dt><strong><code>device</code></strong> : <code>tp.Union[torch.device, str]</code></dt>
|
||
<dd>Device.</dd>
|
||
</dl>
|
||
<p>Initializes internal Module state, shared by both nn.Module and ScriptModule.</p></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.conditioners.BaseConditioner" href="#audiocraft.modules.conditioners.BaseConditioner">BaseConditioner</a></li>
|
||
<li>torch.nn.modules.module.Module</li>
|
||
</ul>
|
||
<h3>Subclasses</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.conditioners.ChromaStemConditioner" href="#audiocraft.modules.conditioners.ChromaStemConditioner">ChromaStemConditioner</a></li>
|
||
<li><a title="audiocraft.modules.conditioners.FeatureExtractor" href="#audiocraft.modules.conditioners.FeatureExtractor">FeatureExtractor</a></li>
|
||
<li><a title="audiocraft.modules.jasco_conditioners.DrumsConditioner" href="jasco_conditioners.html#audiocraft.modules.jasco_conditioners.DrumsConditioner">DrumsConditioner</a></li>
|
||
</ul>
|
||
<h3>Class variables</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.WaveformConditioner.call_super_init"><code class="name">var <span class="ident">call_super_init</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.WaveformConditioner.dump_patches"><code class="name">var <span class="ident">dump_patches</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.WaveformConditioner.training"><code class="name">var <span class="ident">training</span> : bool</code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
<h3>Methods</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.WaveformConditioner.forward"><code class="name flex">
|
||
<span>def <span class="ident">forward</span></span>(<span>self,<br>x: <a title="audiocraft.modules.conditioners.WavCondition" href="#audiocraft.modules.conditioners.WavCondition">WavCondition</a>) ‑> Tuple[torch.Tensor, torch.Tensor]</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def forward(self, x: WavCondition) -> ConditionType:
|
||
"""Extract condition embedding and mask from a waveform and its metadata.
|
||
Args:
|
||
x (WavCondition): Waveform condition containing raw waveform and metadata.
|
||
Returns:
|
||
ConditionType: a dense vector representing the conditioning along with its mask
|
||
"""
|
||
wav, lengths, *_ = x
|
||
with torch.no_grad():
|
||
embeds = self._get_wav_embedding(x)
|
||
if hasattr(self, 'output_proj'):
|
||
embeds = embeds.to(self.output_proj.weight)
|
||
embeds = self.output_proj(embeds)
|
||
|
||
if lengths is not None and self._use_masking:
|
||
lengths = lengths / self._downsampling_factor()
|
||
mask = length_to_mask(lengths, max_len=embeds.shape[1]).int() # type: ignore
|
||
else:
|
||
mask = torch.ones_like(embeds[..., 0])
|
||
embeds = (embeds * mask.unsqueeze(-1))
|
||
return embeds, mask</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Extract condition embedding and mask from a waveform and its metadata.</p>
|
||
<h2 id="args">Args</h2>
|
||
<dl>
|
||
<dt><strong><code>x</code></strong> : <code><a title="audiocraft.modules.conditioners.WavCondition" href="#audiocraft.modules.conditioners.WavCondition">WavCondition</a></code></dt>
|
||
<dd>Waveform condition containing raw waveform and metadata.</dd>
|
||
</dl>
|
||
<h2 id="returns">Returns</h2>
|
||
<dl>
|
||
<dt><code>ConditionType</code></dt>
|
||
<dd>a dense vector representing the conditioning along with its mask</dd>
|
||
</dl></div>
|
||
</dd>
|
||
</dl>
|
||
<h3>Inherited members</h3>
|
||
<ul class="hlist">
|
||
<li><code><b><a title="audiocraft.modules.conditioners.BaseConditioner" href="#audiocraft.modules.conditioners.BaseConditioner">BaseConditioner</a></b></code>:
|
||
<ul class="hlist">
|
||
<li><code><a title="audiocraft.modules.conditioners.BaseConditioner.tokenize" href="#audiocraft.modules.conditioners.BaseConditioner.tokenize">tokenize</a></code></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</dd>
|
||
<dt id="audiocraft.modules.conditioners.WhiteSpaceTokenizer"><code class="flex name class">
|
||
<span>class <span class="ident">WhiteSpaceTokenizer</span></span>
|
||
<span>(</span><span>n_bins: int,<br>pad_idx: int = 0,<br>language: str = 'en_core_web_sm',<br>lemma: bool = True,<br>stopwords: bool = True)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class WhiteSpaceTokenizer(Tokenizer):
|
||
"""This tokenizer should be used for natural language descriptions.
|
||
For example:
|
||
["he didn't, know he's going home.", 'shorter sentence'] =>
|
||
[[78, 62, 31, 4, 78, 25, 19, 34],
|
||
[59, 77, 0, 0, 0, 0, 0, 0]]
|
||
"""
|
||
PUNCTUATION = "?:!.,;"
|
||
|
||
def __init__(self, n_bins: int, pad_idx: int = 0, language: str = "en_core_web_sm",
|
||
lemma: bool = True, stopwords: bool = True) -> None:
|
||
self.n_bins = n_bins
|
||
self.pad_idx = pad_idx
|
||
self.lemma = lemma
|
||
self.stopwords = stopwords
|
||
try:
|
||
self.nlp = spacy.load(language)
|
||
except IOError:
|
||
spacy.cli.download(language) # type: ignore
|
||
self.nlp = spacy.load(language)
|
||
|
||
@tp.no_type_check
|
||
def __call__(self, texts: tp.List[tp.Optional[str]],
|
||
return_text: bool = False) -> tp.Tuple[torch.Tensor, torch.Tensor]:
|
||
"""Take a list of strings and convert them to a tensor of indices.
|
||
|
||
Args:
|
||
texts (list[str]): List of strings.
|
||
return_text (bool, optional): Whether to return text as additional tuple item. Defaults to False.
|
||
Returns:
|
||
tuple[torch.Tensor, torch.Tensor]:
|
||
- Indices of words in the LUT.
|
||
- And a mask indicating where the padding tokens are
|
||
"""
|
||
output, lengths = [], []
|
||
texts = deepcopy(texts)
|
||
for i, text in enumerate(texts):
|
||
# if current sample doesn't have a certain attribute, replace with pad token
|
||
if text is None:
|
||
output.append(torch.Tensor([self.pad_idx]))
|
||
lengths.append(0)
|
||
continue
|
||
|
||
# convert numbers to words
|
||
text = re.sub(r"(\d+)", lambda x: num2words(int(x.group(0))), text) # type: ignore
|
||
# normalize text
|
||
text = self.nlp(text) # type: ignore
|
||
# remove stopwords
|
||
if self.stopwords:
|
||
text = [w for w in text if not w.is_stop] # type: ignore
|
||
# remove punctuation
|
||
text = [w for w in text if w.text not in self.PUNCTUATION] # type: ignore
|
||
# lemmatize if needed
|
||
text = [getattr(t, "lemma_" if self.lemma else "text") for t in text] # type: ignore
|
||
|
||
texts[i] = " ".join(text)
|
||
lengths.append(len(text))
|
||
# convert to tensor
|
||
tokens = torch.Tensor([hash_trick(w, self.n_bins) for w in text])
|
||
output.append(tokens)
|
||
|
||
mask = length_to_mask(torch.IntTensor(lengths)).int()
|
||
padded_output = pad_sequence(output, padding_value=self.pad_idx).int().t()
|
||
if return_text:
|
||
return padded_output, mask, texts # type: ignore
|
||
return padded_output, mask</code></pre>
|
||
</details>
|
||
<div class="desc"><p>This tokenizer should be used for natural language descriptions.
|
||
For example:
|
||
["he didn't, know he's going home.", 'shorter sentence'] =>
|
||
[[78, 62, 31,
|
||
4, 78, 25, 19, 34],
|
||
[59, 77,
|
||
0,
|
||
0,
|
||
0,
|
||
0,
|
||
0,
|
||
0]]</p></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li><a title="audiocraft.modules.conditioners.Tokenizer" href="#audiocraft.modules.conditioners.Tokenizer">Tokenizer</a></li>
|
||
</ul>
|
||
<h3>Class variables</h3>
|
||
<dl>
|
||
<dt id="audiocraft.modules.conditioners.WhiteSpaceTokenizer.PUNCTUATION"><code class="name">var <span class="ident">PUNCTUATION</span></code></dt>
|
||
<dd>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
</dd>
|
||
</dl>
|
||
</section>
|
||
</article>
|
||
<nav id="sidebar">
|
||
<div class="toc">
|
||
<ul></ul>
|
||
</div>
|
||
<ul id="index">
|
||
<li><h3>Super-module</h3>
|
||
<ul>
|
||
<li><code><a title="audiocraft.modules" href="index.html">audiocraft.modules</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li><h3><a href="#header-functions">Functions</a></h3>
|
||
<ul class="">
|
||
<li><code><a title="audiocraft.modules.conditioners.dropout_condition" href="#audiocraft.modules.conditioners.dropout_condition">dropout_condition</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.dropout_symbolic_conditions" href="#audiocraft.modules.conditioners.dropout_symbolic_conditions">dropout_symbolic_conditions</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.nullify_chords" href="#audiocraft.modules.conditioners.nullify_chords">nullify_chords</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.nullify_condition" href="#audiocraft.modules.conditioners.nullify_condition">nullify_condition</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.nullify_joint_embed" href="#audiocraft.modules.conditioners.nullify_joint_embed">nullify_joint_embed</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.nullify_melody" href="#audiocraft.modules.conditioners.nullify_melody">nullify_melody</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.nullify_wav" href="#audiocraft.modules.conditioners.nullify_wav">nullify_wav</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li><h3><a href="#header-classes">Classes</a></h3>
|
||
<ul>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.conditioners.AttributeDropout" href="#audiocraft.modules.conditioners.AttributeDropout">AttributeDropout</a></code></h4>
|
||
<ul class="">
|
||
<li><code><a title="audiocraft.modules.conditioners.AttributeDropout.call_super_init" href="#audiocraft.modules.conditioners.AttributeDropout.call_super_init">call_super_init</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.AttributeDropout.dump_patches" href="#audiocraft.modules.conditioners.AttributeDropout.dump_patches">dump_patches</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.AttributeDropout.forward" href="#audiocraft.modules.conditioners.AttributeDropout.forward">forward</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.AttributeDropout.training" href="#audiocraft.modules.conditioners.AttributeDropout.training">training</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.conditioners.BaseConditioner" href="#audiocraft.modules.conditioners.BaseConditioner">BaseConditioner</a></code></h4>
|
||
<ul class="">
|
||
<li><code><a title="audiocraft.modules.conditioners.BaseConditioner.call_super_init" href="#audiocraft.modules.conditioners.BaseConditioner.call_super_init">call_super_init</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.BaseConditioner.dump_patches" href="#audiocraft.modules.conditioners.BaseConditioner.dump_patches">dump_patches</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.BaseConditioner.forward" href="#audiocraft.modules.conditioners.BaseConditioner.forward">forward</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.BaseConditioner.tokenize" href="#audiocraft.modules.conditioners.BaseConditioner.tokenize">tokenize</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.BaseConditioner.training" href="#audiocraft.modules.conditioners.BaseConditioner.training">training</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.conditioners.CLAPEmbeddingConditioner" href="#audiocraft.modules.conditioners.CLAPEmbeddingConditioner">CLAPEmbeddingConditioner</a></code></h4>
|
||
<ul class="">
|
||
<li><code><a title="audiocraft.modules.conditioners.CLAPEmbeddingConditioner.call_super_init" href="#audiocraft.modules.conditioners.CLAPEmbeddingConditioner.call_super_init">call_super_init</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.CLAPEmbeddingConditioner.dump_patches" href="#audiocraft.modules.conditioners.CLAPEmbeddingConditioner.dump_patches">dump_patches</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.CLAPEmbeddingConditioner.training" href="#audiocraft.modules.conditioners.CLAPEmbeddingConditioner.training">training</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.conditioners.ChromaStemConditioner" href="#audiocraft.modules.conditioners.ChromaStemConditioner">ChromaStemConditioner</a></code></h4>
|
||
<ul class="two-column">
|
||
<li><code><a title="audiocraft.modules.conditioners.ChromaStemConditioner.call_super_init" href="#audiocraft.modules.conditioners.ChromaStemConditioner.call_super_init">call_super_init</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ChromaStemConditioner.dump_patches" href="#audiocraft.modules.conditioners.ChromaStemConditioner.dump_patches">dump_patches</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ChromaStemConditioner.has_eval_wavs" href="#audiocraft.modules.conditioners.ChromaStemConditioner.has_eval_wavs">has_eval_wavs</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ChromaStemConditioner.reset_eval_wavs" href="#audiocraft.modules.conditioners.ChromaStemConditioner.reset_eval_wavs">reset_eval_wavs</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ChromaStemConditioner.tokenize" href="#audiocraft.modules.conditioners.ChromaStemConditioner.tokenize">tokenize</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ChromaStemConditioner.training" href="#audiocraft.modules.conditioners.ChromaStemConditioner.training">training</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.conditioners.ClassifierFreeGuidanceDropout" href="#audiocraft.modules.conditioners.ClassifierFreeGuidanceDropout">ClassifierFreeGuidanceDropout</a></code></h4>
|
||
<ul class="">
|
||
<li><code><a title="audiocraft.modules.conditioners.ClassifierFreeGuidanceDropout.call_super_init" href="#audiocraft.modules.conditioners.ClassifierFreeGuidanceDropout.call_super_init">call_super_init</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ClassifierFreeGuidanceDropout.dump_patches" href="#audiocraft.modules.conditioners.ClassifierFreeGuidanceDropout.dump_patches">dump_patches</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ClassifierFreeGuidanceDropout.forward" href="#audiocraft.modules.conditioners.ClassifierFreeGuidanceDropout.forward">forward</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ClassifierFreeGuidanceDropout.training" href="#audiocraft.modules.conditioners.ClassifierFreeGuidanceDropout.training">training</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.conditioners.ConditionFuser" href="#audiocraft.modules.conditioners.ConditionFuser">ConditionFuser</a></code></h4>
|
||
<ul class="">
|
||
<li><code><a title="audiocraft.modules.conditioners.ConditionFuser.FUSING_METHODS" href="#audiocraft.modules.conditioners.ConditionFuser.FUSING_METHODS">FUSING_METHODS</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ConditionFuser.call_super_init" href="#audiocraft.modules.conditioners.ConditionFuser.call_super_init">call_super_init</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ConditionFuser.dump_patches" href="#audiocraft.modules.conditioners.ConditionFuser.dump_patches">dump_patches</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ConditionFuser.forward" href="#audiocraft.modules.conditioners.ConditionFuser.forward">forward</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ConditionFuser.training" href="#audiocraft.modules.conditioners.ConditionFuser.training">training</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.conditioners.ConditioningAttributes" href="#audiocraft.modules.conditioners.ConditioningAttributes">ConditioningAttributes</a></code></h4>
|
||
<ul class="">
|
||
<li><code><a title="audiocraft.modules.conditioners.ConditioningAttributes.attributes" href="#audiocraft.modules.conditioners.ConditioningAttributes.attributes">attributes</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ConditioningAttributes.from_flat_dict" href="#audiocraft.modules.conditioners.ConditioningAttributes.from_flat_dict">from_flat_dict</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ConditioningAttributes.joint_embed" href="#audiocraft.modules.conditioners.ConditioningAttributes.joint_embed">joint_embed</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ConditioningAttributes.joint_embed_attributes" href="#audiocraft.modules.conditioners.ConditioningAttributes.joint_embed_attributes">joint_embed_attributes</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ConditioningAttributes.symbolic" href="#audiocraft.modules.conditioners.ConditioningAttributes.symbolic">symbolic</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ConditioningAttributes.symbolic_attributes" href="#audiocraft.modules.conditioners.ConditioningAttributes.symbolic_attributes">symbolic_attributes</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ConditioningAttributes.text" href="#audiocraft.modules.conditioners.ConditioningAttributes.text">text</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ConditioningAttributes.text_attributes" href="#audiocraft.modules.conditioners.ConditioningAttributes.text_attributes">text_attributes</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ConditioningAttributes.to_flat_dict" href="#audiocraft.modules.conditioners.ConditioningAttributes.to_flat_dict">to_flat_dict</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ConditioningAttributes.wav" href="#audiocraft.modules.conditioners.ConditioningAttributes.wav">wav</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ConditioningAttributes.wav_attributes" href="#audiocraft.modules.conditioners.ConditioningAttributes.wav_attributes">wav_attributes</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.conditioners.ConditioningProvider" href="#audiocraft.modules.conditioners.ConditioningProvider">ConditioningProvider</a></code></h4>
|
||
<ul class="">
|
||
<li><code><a title="audiocraft.modules.conditioners.ConditioningProvider.call_super_init" href="#audiocraft.modules.conditioners.ConditioningProvider.call_super_init">call_super_init</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ConditioningProvider.dump_patches" href="#audiocraft.modules.conditioners.ConditioningProvider.dump_patches">dump_patches</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ConditioningProvider.forward" href="#audiocraft.modules.conditioners.ConditioningProvider.forward">forward</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ConditioningProvider.has_joint_embed_conditions" href="#audiocraft.modules.conditioners.ConditioningProvider.has_joint_embed_conditions">has_joint_embed_conditions</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ConditioningProvider.has_wav_condition" href="#audiocraft.modules.conditioners.ConditioningProvider.has_wav_condition">has_wav_condition</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ConditioningProvider.joint_embed_conditions" href="#audiocraft.modules.conditioners.ConditioningProvider.joint_embed_conditions">joint_embed_conditions</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ConditioningProvider.text_conditions" href="#audiocraft.modules.conditioners.ConditioningProvider.text_conditions">text_conditions</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ConditioningProvider.tokenize" href="#audiocraft.modules.conditioners.ConditioningProvider.tokenize">tokenize</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ConditioningProvider.training" href="#audiocraft.modules.conditioners.ConditioningProvider.training">training</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.ConditioningProvider.wav_conditions" href="#audiocraft.modules.conditioners.ConditioningProvider.wav_conditions">wav_conditions</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.conditioners.DropoutModule" href="#audiocraft.modules.conditioners.DropoutModule">DropoutModule</a></code></h4>
|
||
<ul class="">
|
||
<li><code><a title="audiocraft.modules.conditioners.DropoutModule.call_super_init" href="#audiocraft.modules.conditioners.DropoutModule.call_super_init">call_super_init</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.DropoutModule.dump_patches" href="#audiocraft.modules.conditioners.DropoutModule.dump_patches">dump_patches</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.DropoutModule.forward" href="#audiocraft.modules.conditioners.DropoutModule.forward">forward</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.DropoutModule.training" href="#audiocraft.modules.conditioners.DropoutModule.training">training</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.conditioners.FeatureExtractor" href="#audiocraft.modules.conditioners.FeatureExtractor">FeatureExtractor</a></code></h4>
|
||
<ul class="">
|
||
<li><code><a title="audiocraft.modules.conditioners.FeatureExtractor.call_super_init" href="#audiocraft.modules.conditioners.FeatureExtractor.call_super_init">call_super_init</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.FeatureExtractor.dump_patches" href="#audiocraft.modules.conditioners.FeatureExtractor.dump_patches">dump_patches</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.FeatureExtractor.training" href="#audiocraft.modules.conditioners.FeatureExtractor.training">training</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.conditioners.JascoCondConst" href="#audiocraft.modules.conditioners.JascoCondConst">JascoCondConst</a></code></h4>
|
||
<ul class="two-column">
|
||
<li><code><a title="audiocraft.modules.conditioners.JascoCondConst.ALL" href="#audiocraft.modules.conditioners.JascoCondConst.ALL">ALL</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.JascoCondConst.CRD" href="#audiocraft.modules.conditioners.JascoCondConst.CRD">CRD</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.JascoCondConst.DRM" href="#audiocraft.modules.conditioners.JascoCondConst.DRM">DRM</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.JascoCondConst.LAT" href="#audiocraft.modules.conditioners.JascoCondConst.LAT">LAT</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.JascoCondConst.MLD" href="#audiocraft.modules.conditioners.JascoCondConst.MLD">MLD</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.JascoCondConst.SYM" href="#audiocraft.modules.conditioners.JascoCondConst.SYM">SYM</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.conditioners.JointEmbedCondition" href="#audiocraft.modules.conditioners.JointEmbedCondition">JointEmbedCondition</a></code></h4>
|
||
<ul class="two-column">
|
||
<li><code><a title="audiocraft.modules.conditioners.JointEmbedCondition.length" href="#audiocraft.modules.conditioners.JointEmbedCondition.length">length</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.JointEmbedCondition.path" href="#audiocraft.modules.conditioners.JointEmbedCondition.path">path</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.JointEmbedCondition.sample_rate" href="#audiocraft.modules.conditioners.JointEmbedCondition.sample_rate">sample_rate</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.JointEmbedCondition.seek_time" href="#audiocraft.modules.conditioners.JointEmbedCondition.seek_time">seek_time</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.JointEmbedCondition.text" href="#audiocraft.modules.conditioners.JointEmbedCondition.text">text</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.JointEmbedCondition.wav" href="#audiocraft.modules.conditioners.JointEmbedCondition.wav">wav</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.conditioners.JointEmbeddingConditioner" href="#audiocraft.modules.conditioners.JointEmbeddingConditioner">JointEmbeddingConditioner</a></code></h4>
|
||
<ul class="">
|
||
<li><code><a title="audiocraft.modules.conditioners.JointEmbeddingConditioner.call_super_init" href="#audiocraft.modules.conditioners.JointEmbeddingConditioner.call_super_init">call_super_init</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.JointEmbeddingConditioner.dump_patches" href="#audiocraft.modules.conditioners.JointEmbeddingConditioner.dump_patches">dump_patches</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.JointEmbeddingConditioner.training" href="#audiocraft.modules.conditioners.JointEmbeddingConditioner.training">training</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.conditioners.LUTConditioner" href="#audiocraft.modules.conditioners.LUTConditioner">LUTConditioner</a></code></h4>
|
||
<ul class="">
|
||
<li><code><a title="audiocraft.modules.conditioners.LUTConditioner.call_super_init" href="#audiocraft.modules.conditioners.LUTConditioner.call_super_init">call_super_init</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.LUTConditioner.dump_patches" href="#audiocraft.modules.conditioners.LUTConditioner.dump_patches">dump_patches</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.LUTConditioner.training" href="#audiocraft.modules.conditioners.LUTConditioner.training">training</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.conditioners.NoopTokenizer" href="#audiocraft.modules.conditioners.NoopTokenizer">NoopTokenizer</a></code></h4>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.conditioners.SegmentWithAttributes" href="#audiocraft.modules.conditioners.SegmentWithAttributes">SegmentWithAttributes</a></code></h4>
|
||
<ul class="">
|
||
<li><code><a title="audiocraft.modules.conditioners.SegmentWithAttributes.channels" href="#audiocraft.modules.conditioners.SegmentWithAttributes.channels">channels</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.SegmentWithAttributes.meta" href="#audiocraft.modules.conditioners.SegmentWithAttributes.meta">meta</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.SegmentWithAttributes.n_frames" href="#audiocraft.modules.conditioners.SegmentWithAttributes.n_frames">n_frames</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.SegmentWithAttributes.sample_rate" href="#audiocraft.modules.conditioners.SegmentWithAttributes.sample_rate">sample_rate</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.SegmentWithAttributes.seek_time" href="#audiocraft.modules.conditioners.SegmentWithAttributes.seek_time">seek_time</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.SegmentWithAttributes.to_condition_attributes" href="#audiocraft.modules.conditioners.SegmentWithAttributes.to_condition_attributes">to_condition_attributes</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.SegmentWithAttributes.total_frames" href="#audiocraft.modules.conditioners.SegmentWithAttributes.total_frames">total_frames</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.conditioners.StyleConditioner" href="#audiocraft.modules.conditioners.StyleConditioner">StyleConditioner</a></code></h4>
|
||
<ul class="">
|
||
<li><code><a title="audiocraft.modules.conditioners.StyleConditioner.call_super_init" href="#audiocraft.modules.conditioners.StyleConditioner.call_super_init">call_super_init</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.StyleConditioner.dump_patches" href="#audiocraft.modules.conditioners.StyleConditioner.dump_patches">dump_patches</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.StyleConditioner.set_params" href="#audiocraft.modules.conditioners.StyleConditioner.set_params">set_params</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.StyleConditioner.training" href="#audiocraft.modules.conditioners.StyleConditioner.training">training</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.conditioners.SymbolicCondition" href="#audiocraft.modules.conditioners.SymbolicCondition">SymbolicCondition</a></code></h4>
|
||
<ul class="">
|
||
<li><code><a title="audiocraft.modules.conditioners.SymbolicCondition.frame_chords" href="#audiocraft.modules.conditioners.SymbolicCondition.frame_chords">frame_chords</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.SymbolicCondition.melody" href="#audiocraft.modules.conditioners.SymbolicCondition.melody">melody</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.conditioners.T5Conditioner" href="#audiocraft.modules.conditioners.T5Conditioner">T5Conditioner</a></code></h4>
|
||
<ul class="">
|
||
<li><code><a title="audiocraft.modules.conditioners.T5Conditioner.MODELS" href="#audiocraft.modules.conditioners.T5Conditioner.MODELS">MODELS</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.T5Conditioner.MODELS_DIMS" href="#audiocraft.modules.conditioners.T5Conditioner.MODELS_DIMS">MODELS_DIMS</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.T5Conditioner.call_super_init" href="#audiocraft.modules.conditioners.T5Conditioner.call_super_init">call_super_init</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.T5Conditioner.dump_patches" href="#audiocraft.modules.conditioners.T5Conditioner.dump_patches">dump_patches</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.T5Conditioner.training" href="#audiocraft.modules.conditioners.T5Conditioner.training">training</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.conditioners.TextConditioner" href="#audiocraft.modules.conditioners.TextConditioner">TextConditioner</a></code></h4>
|
||
<ul class="">
|
||
<li><code><a title="audiocraft.modules.conditioners.TextConditioner.call_super_init" href="#audiocraft.modules.conditioners.TextConditioner.call_super_init">call_super_init</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.TextConditioner.dump_patches" href="#audiocraft.modules.conditioners.TextConditioner.dump_patches">dump_patches</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.TextConditioner.training" href="#audiocraft.modules.conditioners.TextConditioner.training">training</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.conditioners.Tokenizer" href="#audiocraft.modules.conditioners.Tokenizer">Tokenizer</a></code></h4>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.conditioners.WavCondition" href="#audiocraft.modules.conditioners.WavCondition">WavCondition</a></code></h4>
|
||
<ul class="">
|
||
<li><code><a title="audiocraft.modules.conditioners.WavCondition.length" href="#audiocraft.modules.conditioners.WavCondition.length">length</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.WavCondition.path" href="#audiocraft.modules.conditioners.WavCondition.path">path</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.WavCondition.sample_rate" href="#audiocraft.modules.conditioners.WavCondition.sample_rate">sample_rate</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.WavCondition.seek_time" href="#audiocraft.modules.conditioners.WavCondition.seek_time">seek_time</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.WavCondition.wav" href="#audiocraft.modules.conditioners.WavCondition.wav">wav</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.conditioners.WaveformConditioner" href="#audiocraft.modules.conditioners.WaveformConditioner">WaveformConditioner</a></code></h4>
|
||
<ul class="">
|
||
<li><code><a title="audiocraft.modules.conditioners.WaveformConditioner.call_super_init" href="#audiocraft.modules.conditioners.WaveformConditioner.call_super_init">call_super_init</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.WaveformConditioner.dump_patches" href="#audiocraft.modules.conditioners.WaveformConditioner.dump_patches">dump_patches</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.WaveformConditioner.forward" href="#audiocraft.modules.conditioners.WaveformConditioner.forward">forward</a></code></li>
|
||
<li><code><a title="audiocraft.modules.conditioners.WaveformConditioner.training" href="#audiocraft.modules.conditioners.WaveformConditioner.training">training</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="audiocraft.modules.conditioners.WhiteSpaceTokenizer" href="#audiocraft.modules.conditioners.WhiteSpaceTokenizer">WhiteSpaceTokenizer</a></code></h4>
|
||
<ul class="">
|
||
<li><code><a title="audiocraft.modules.conditioners.WhiteSpaceTokenizer.PUNCTUATION" href="#audiocraft.modules.conditioners.WhiteSpaceTokenizer.PUNCTUATION">PUNCTUATION</a></code></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</nav>
|
||
</main>
|
||
<footer id="footer">
|
||
<p>Generated by <a href="https://pdoc3.github.io/pdoc" title="pdoc: Python API documentation generator"><cite>pdoc</cite> 0.11.5</a>.</p>
|
||
</footer>
|
||
</body>
|
||
</html>
|