Spaces:
Running on Zero
Running on Zero
Add six official Anima bases with reference KV and Turbo inference
Browse files- LICENSE-CODE.txt +202 -0
- NOTICE.txt +21 -0
- README.md +77 -7
- anima_role_ref_kv/__init__.py +5 -0
- anima_role_ref_kv/components.py +94 -0
- anima_role_ref_kv/lora.py +60 -0
- anima_role_ref_kv/pipeline.py +51 -0
- anima_role_ref_kv/reference.py +129 -0
- anima_role_ref_kv/weights.py +55 -0
- app.py +154 -0
- checkpoints.py +102 -0
- models.py +45 -0
- requirements.txt +8 -0
- runtime.py +51 -0
- smoke.py +68 -0
- tests/test_space.py +90 -0
LICENSE-CODE.txt
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Apache License
|
| 2 |
+
Version 2.0, January 2004
|
| 3 |
+
http://www.apache.org/licenses/
|
| 4 |
+
|
| 5 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 6 |
+
|
| 7 |
+
1. Definitions.
|
| 8 |
+
|
| 9 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 10 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 11 |
+
|
| 12 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 13 |
+
the copyright owner that is granting the License.
|
| 14 |
+
|
| 15 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 16 |
+
other entities that control, are controlled by, or are under common
|
| 17 |
+
control with that entity. For the purposes of this definition,
|
| 18 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 19 |
+
direction or management of such entity, whether by contract or
|
| 20 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 21 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 22 |
+
|
| 23 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 24 |
+
exercising permissions granted by this License.
|
| 25 |
+
|
| 26 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 27 |
+
including but not limited to software source code, documentation
|
| 28 |
+
source, and configuration files.
|
| 29 |
+
|
| 30 |
+
"Object" form shall mean any form resulting from mechanical
|
| 31 |
+
transformation or translation of a Source form, including but
|
| 32 |
+
not limited to compiled object code, generated documentation,
|
| 33 |
+
and conversions to other media types.
|
| 34 |
+
|
| 35 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 36 |
+
Object form, made available under the License, as indicated by a
|
| 37 |
+
copyright notice that is included in or attached to the work
|
| 38 |
+
(an example is provided in the Appendix below).
|
| 39 |
+
|
| 40 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 41 |
+
form, that is based on (or derived from) the Work and for which the
|
| 42 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 43 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 44 |
+
of this License, Derivative Works shall not include works that remain
|
| 45 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 46 |
+
the Work and Derivative Works thereof.
|
| 47 |
+
|
| 48 |
+
"Contribution" shall mean any work of authorship, including
|
| 49 |
+
the original version of the Work and any modifications or additions
|
| 50 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 51 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 52 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 53 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 54 |
+
means any form of electronic, verbal, or written communication sent
|
| 55 |
+
to the Licensor or its representatives, including but not limited to
|
| 56 |
+
communication on electronic mailing lists, source code control systems,
|
| 57 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 58 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 59 |
+
excluding communication that is conspicuously marked or otherwise
|
| 60 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 61 |
+
|
| 62 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 63 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 64 |
+
subsequently incorporated within the Work.
|
| 65 |
+
|
| 66 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 67 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 68 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 69 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 70 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 71 |
+
Work and such Derivative Works in Source or Object form.
|
| 72 |
+
|
| 73 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 74 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 75 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 76 |
+
(except as stated in this section) patent license to make, have made,
|
| 77 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 78 |
+
where such license applies only to those patent claims licensable
|
| 79 |
+
by such Contributor that are necessarily infringed by their
|
| 80 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 81 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 82 |
+
institute patent litigation against any entity (including a
|
| 83 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 84 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 85 |
+
or contributory patent infringement, then any patent licenses
|
| 86 |
+
granted to You under this License for that Work shall terminate
|
| 87 |
+
as of the date such litigation is filed.
|
| 88 |
+
|
| 89 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 90 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 91 |
+
modifications, and in Source or Object form, provided that You
|
| 92 |
+
meet the following conditions:
|
| 93 |
+
|
| 94 |
+
(a) You must give any other recipients of the Work or
|
| 95 |
+
Derivative Works a copy of this License; and
|
| 96 |
+
|
| 97 |
+
(b) You must cause any modified files to carry prominent notices
|
| 98 |
+
stating that You changed the files; and
|
| 99 |
+
|
| 100 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 101 |
+
that You distribute, all copyright, patent, trademark, and
|
| 102 |
+
attribution notices from the Source form of the Work,
|
| 103 |
+
excluding those notices that do not pertain to any part of
|
| 104 |
+
the Derivative Works; and
|
| 105 |
+
|
| 106 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 107 |
+
distribution, then any Derivative Works that You distribute must
|
| 108 |
+
include a readable copy of the attribution notices contained
|
| 109 |
+
within such NOTICE file, excluding those notices that do not
|
| 110 |
+
pertain to any part of the Derivative Works, in at least one
|
| 111 |
+
of the following places: within a NOTICE text file distributed
|
| 112 |
+
as part of the Derivative Works; within the Source form or
|
| 113 |
+
documentation, if provided along with the Derivative Works; or,
|
| 114 |
+
within a display generated by the Derivative Works, if and
|
| 115 |
+
wherever such third-party notices normally appear. The contents
|
| 116 |
+
of the NOTICE file are for informational purposes only and
|
| 117 |
+
do not modify the License. You may add Your own attribution
|
| 118 |
+
notices within Derivative Works that You distribute, alongside
|
| 119 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 120 |
+
that such additional attribution notices cannot be construed
|
| 121 |
+
as modifying the License.
|
| 122 |
+
|
| 123 |
+
You may add Your own copyright statement to Your modifications and
|
| 124 |
+
may provide additional or different license terms and conditions
|
| 125 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 126 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 127 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 128 |
+
the conditions stated in this License.
|
| 129 |
+
|
| 130 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 131 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 132 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 133 |
+
this License, without any additional terms or conditions.
|
| 134 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 135 |
+
the terms of any separate license agreement you may have executed
|
| 136 |
+
with Licensor regarding such Contributions.
|
| 137 |
+
|
| 138 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 139 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 140 |
+
except as required for reasonable and customary use in describing the
|
| 141 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 142 |
+
|
| 143 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 144 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 145 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 146 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 147 |
+
implied, including, without limitation, Any warranties or conditions
|
| 148 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 149 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 150 |
+
appropriateness of using or redistributing the Work and assume any
|
| 151 |
+
risks associated with Your exercise of permissions under this License.
|
| 152 |
+
|
| 153 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 154 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 155 |
+
unless required by applicable law (such as deliberate and grossly
|
| 156 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 157 |
+
liable to You for damages, including any direct, indirect, special,
|
| 158 |
+
incidental, or consequential damages of any character arising as a
|
| 159 |
+
result of this License or out of the use or inability to use the
|
| 160 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 161 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 162 |
+
other commercial damages or losses), even if such Contributor
|
| 163 |
+
has been advised of the possibility of such damages.
|
| 164 |
+
|
| 165 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 166 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 167 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 168 |
+
or other liability obligations and/or rights consistent with this
|
| 169 |
+
License. However, in accepting such obligations, You may act only
|
| 170 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 171 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 172 |
+
defend, and hold each Contributor harmless for any liability
|
| 173 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 174 |
+
of your accepting any such warranty or additional liability.
|
| 175 |
+
|
| 176 |
+
END OF TERMS AND CONDITIONS
|
| 177 |
+
|
| 178 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 179 |
+
|
| 180 |
+
To apply the Apache License to your work, attach the following
|
| 181 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 182 |
+
replaced with your own identifying information. (Don't include
|
| 183 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 184 |
+
comment syntax for the file format. We also recommend that a
|
| 185 |
+
file or class name and description of purpose be included on the
|
| 186 |
+
same "printed page" as the copyright notice for easier
|
| 187 |
+
identification within third-party archives.
|
| 188 |
+
|
| 189 |
+
Copyright [yyyy] [name of copyright owner]
|
| 190 |
+
|
| 191 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 192 |
+
you may not use this file except in compliance with the License.
|
| 193 |
+
You may obtain a copy of the License at
|
| 194 |
+
|
| 195 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 196 |
+
|
| 197 |
+
Unless required by applicable law or agreed to in writing, software
|
| 198 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 199 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 200 |
+
See the License for the specific language governing permissions and
|
| 201 |
+
limitations under the License.
|
| 202 |
+
|
NOTICE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Anima Role Ref KV v1.0 is a modified derivative of Anima Base v1.0.
|
| 2 |
+
This independent release is not endorsed, approved, or validated by CircleStone Labs.
|
| 3 |
+
|
| 4 |
+
The CircleStone Model is licensed by CircleStone Labs LLC under the CircleStone Non-Commercial License. Copyright CircleStone Labs LLC.
|
| 5 |
+
IN NO EVENT SHALL CIRCLESTONE LABS LLC BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH USE OF THIS MODEL.
|
| 6 |
+
|
| 7 |
+
Rights to the CircleStone Model and its derivatives are granted directly by CircleStone Labs under LICENSE-MODEL.txt. The model license is preserved from the base model distribution used for this release. Code licenses do not replace or expand the model license.
|
| 8 |
+
|
| 9 |
+
Python inference code in anima_role_ref_kv/ and infer.py: Apache License 2.0 (LICENSE-CODE.txt).
|
| 10 |
+
Parts of anima_role_ref_kv/reference.py are adapted from Hugging Face Diffusers transformer_cosmos.py, Copyright the Hugging Face team, under Apache License 2.0. These files are modified for this release.
|
| 11 |
+
Diffusers source: https://github.com/huggingface/diffusers/tree/5ff8e59ff9fe81c6e2df4fb4c6ea0d97a5df5ab2
|
| 12 |
+
|
| 13 |
+
ComfyUI integration in comfyui/: GNU General Public License v3.0 (comfyui/LICENSE).
|
| 14 |
+
The integration follows the native Anima/Cosmos model interfaces in ComfyUI:
|
| 15 |
+
https://github.com/Comfy-Org/ComfyUI/tree/cdf272f3514353aab04dc2d9fceef57a260dc156
|
| 16 |
+
ComfyUI includes NVIDIA Cosmos components under Apache License 2.0. Their original notices are retained in the upstream project.
|
| 17 |
+
|
| 18 |
+
All provided code is supplied without warranties; consult the respective full licenses.
|
| 19 |
+
|
| 20 |
+
Space modifications (2026-10-29): official checkpoint switching, configurable adapter strength, progress callbacks and CFG 1 inference optimization. New Space code is Apache-2.0.
|
| 21 |
+
UI inspired by hugging-apps/anima-role-ref-kv-2 (f184fc3087113e6c13129e16d20179525d83d383), by Apolinario / Hugging Face.
|
README.md
CHANGED
|
@@ -1,13 +1,83 @@
|
|
| 1 |
---
|
| 2 |
-
title: Anima Role Ref KV
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
sdk: gradio
|
| 7 |
sdk_version: 6.31.0
|
| 8 |
-
python_version: '3.12'
|
| 9 |
app_file: app.py
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
---
|
| 12 |
|
| 13 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: Anima Role Ref KV · Multi Base
|
| 3 |
+
emoji: 🎭
|
| 4 |
+
colorFrom: purple
|
| 5 |
+
colorTo: pink
|
| 6 |
sdk: gradio
|
| 7 |
sdk_version: 6.31.0
|
|
|
|
| 8 |
app_file: app.py
|
| 9 |
+
python_version: "3.12"
|
| 10 |
+
suggested_hardware: zero-a10g
|
| 11 |
+
startup_duration_timeout: 30m
|
| 12 |
+
license: other
|
| 13 |
+
license_name: circlestone-labs-non-commercial-license
|
| 14 |
+
license_link: https://huggingface.co/circlestone-labs/Anima/blob/main/LICENSE.md
|
| 15 |
+
models:
|
| 16 |
+
- AnticPan/Anima-Role-Ref-KV
|
| 17 |
+
- circlestone-labs/Anima
|
| 18 |
+
- circlestone-labs/Anima-Base-v1.0-Diffusers
|
| 19 |
+
tags:
|
| 20 |
+
- image-to-image
|
| 21 |
+
- character-reference
|
| 22 |
+
- lora
|
| 23 |
+
- anima
|
| 24 |
+
short_description: Character reference with Base, Aesthetic and Turbo
|
| 25 |
---
|
| 26 |
|
| 27 |
+
# Anima Role Ref KV · Multi Base
|
| 28 |
+
|
| 29 |
+
Upload a character reference and select any of six official Anima checkpoints:
|
| 30 |
+
|
| 31 |
+
| Base | Default steps | CFG |
|
| 32 |
+
|---|---:|---:|
|
| 33 |
+
| Base v1.0 | 28 | 4 |
|
| 34 |
+
| Aesthetic v1.0 / v1.0b / v1.1 | 28 | 4 |
|
| 35 |
+
| Turbo v1.0 / v1.1 | 8 | 1 |
|
| 36 |
+
|
| 37 |
+
The published Role Ref KV adapter is applied to the selected base at runtime.
|
| 38 |
+
This is a generation demo, not a downloadable checkpoint merger. The reference
|
| 39 |
+
KV attention runtime is required even if LoRA deltas are fused into weights.
|
| 40 |
+
Base v1.0 is validated by the adapter release; the other combinations are
|
| 41 |
+
experimental. Smoke tests establish inference compatibility, not quality parity.
|
| 42 |
+
|
| 43 |
+
## Implementation
|
| 44 |
+
|
| 45 |
+
- One resident backbone, text encoder, conditioner, VAE and adapter. Switching
|
| 46 |
+
bases replaces original backbone **and text-conditioner** weights, preserving
|
| 47 |
+
the adapter separately. Repeated switches never accumulate LoRA deltas.
|
| 48 |
+
- Official single-file weights map strictly to native Diffusers keys. All keys
|
| 49 |
+
and shapes are checked before modifying the model. Unknown/missing weights fail.
|
| 50 |
+
- The checkpoint downloads run on CPU before GPU allocation and are cached on
|
| 51 |
+
disk. ZeroGPU initializes CUDA tensors at startup; inference uses `spaces.GPU`.
|
| 52 |
+
Non-base requests copy selected weights into those existing CUDA tensors.
|
| 53 |
+
- CFG 1 skips negative text encoding and unconditional denoising. Reference KV
|
| 54 |
+
remains active. All requests use Euler flow sampling, shift 3.
|
| 55 |
+
- Gradio serializes generation because the pipeline is mutable. Every generation
|
| 56 |
+
returns the actual seed, parameters and pinned source revisions.
|
| 57 |
+
- `models.py` pins every model revision. No weights or credentials are committed.
|
| 58 |
+
|
| 59 |
+
## Run
|
| 60 |
+
|
| 61 |
+
Use Python 3.12, install `requirements.txt`, plus `gradio==6.31.0` and `spaces`,
|
| 62 |
+
then run `python app.py` on a CUDA machine. Hugging Face Spaces installs Gradio
|
| 63 |
+
and Spaces automatically. Select ZeroGPU in the Space hardware settings.
|
| 64 |
+
|
| 65 |
+
Optional local environment variables: `ANIMA_BASE_DIR` and `ANIMA_ADAPTER_FILE`.
|
| 66 |
+
They are deployment settings, never public API inputs. Runtime downloads use the
|
| 67 |
+
official Hugging Face repositories. Plan for roughly 30 GB of disk after every
|
| 68 |
+
base has been used; the download cache is ephemeral on a standard Space.
|
| 69 |
+
|
| 70 |
+
Run tests with `python -m unittest discover -s tests -v`. `smoke.py` validates
|
| 71 |
+
real GPU generation and switching with explicitly supplied local checkpoints.
|
| 72 |
+
|
| 73 |
+
## Attribution
|
| 74 |
+
|
| 75 |
+
Based on the author's Apache-2.0 release runtime and inspired by the
|
| 76 |
+
[original Hugging Face demo](https://huggingface.co/spaces/hugging-apps/anima-role-ref-kv-2)
|
| 77 |
+
created by Apolinario and the Hugging Face open-source team. Runtime modifications
|
| 78 |
+
add model switching, adapter-strength control, progress and CFG 1 optimization.
|
| 79 |
+
|
| 80 |
+
An independent modified derivative of Anima, not an official CircleStone Labs
|
| 81 |
+
product. Model use is governed by the CircleStone Labs Non-Commercial License.
|
| 82 |
+
Code licenses and upstream attribution are preserved in NOTICE.txt and
|
| 83 |
+
LICENSE-CODE.txt. No example reference images or user uploads are published.
|
anima_role_ref_kv/__init__.py
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Anima Role Ref KV inference package."""
|
| 2 |
+
|
| 3 |
+
from .pipeline import AnimaRoleRefKVPipeline
|
| 4 |
+
|
| 5 |
+
__all__ = ["AnimaRoleRefKVPipeline"]
|
anima_role_ref_kv/components.py
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Anima model components used by Anima Role Ref KV."""
|
| 2 |
+
|
| 3 |
+
from types import SimpleNamespace
|
| 4 |
+
|
| 5 |
+
import numpy as np
|
| 6 |
+
import torch
|
| 7 |
+
from PIL import Image
|
| 8 |
+
|
| 9 |
+
MODEL = "Anima-Base-v1.0-Diffusers"
|
| 10 |
+
NEGATIVE = ("worst quality, low quality, score_1, score_2, score_3, artist name, blurry, "
|
| 11 |
+
"jpeg artifacts, chromatic aberration")
|
| 12 |
+
STEPS, SHIFT, GUIDANCE = 28, 3.0, 4.0
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def load_transformer(path=MODEL, device="cuda", dtype=torch.bfloat16):
|
| 16 |
+
from diffusers import CosmosTransformer3DModel
|
| 17 |
+
|
| 18 |
+
net, info = CosmosTransformer3DModel.from_pretrained(
|
| 19 |
+
path, subfolder="transformer", torch_dtype=dtype, local_files_only=True, output_loading_info=True)
|
| 20 |
+
if any(info.get(k) for k in ("missing_keys", "unexpected_keys", "mismatched_keys", "error_msgs")):
|
| 21 |
+
raise ValueError(f"Transformer strict loading failed: {info}")
|
| 22 |
+
if net.config.num_layers != 28 or net.config.in_channels != 16 or tuple(net.config.patch_size) != (1, 2, 2):
|
| 23 |
+
raise ValueError("Expected the 28-block Anima base")
|
| 24 |
+
return net.to(device).eval().requires_grad_(False)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def schedule(steps=STEPS, shift=SHIFT, device="cpu"):
|
| 28 |
+
s = torch.linspace(1, 0, steps + 1, device=device, dtype=torch.float32)
|
| 29 |
+
return shift * s / (1 + (shift - 1) * s)
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class PromptEncoder:
|
| 33 |
+
def __init__(self, path=MODEL, device="cuda", dtype=torch.bfloat16, max_length=512):
|
| 34 |
+
from diffusers import AnimaTextConditioner
|
| 35 |
+
from transformers import Qwen2Tokenizer, Qwen3Model, T5Tokenizer
|
| 36 |
+
|
| 37 |
+
self.device, self.dtype, self.max_length = torch.device(device), dtype, max_length
|
| 38 |
+
kw = dict(local_files_only=True)
|
| 39 |
+
self.components = SimpleNamespace(
|
| 40 |
+
tokenizer=Qwen2Tokenizer.from_pretrained(path, subfolder="tokenizer", **kw),
|
| 41 |
+
t5_tokenizer=T5Tokenizer.from_pretrained(path, subfolder="t5_tokenizer", **kw),
|
| 42 |
+
text_encoder=Qwen3Model.from_pretrained(path, subfolder="text_encoder", dtype=dtype, **kw)
|
| 43 |
+
.eval().requires_grad_(False).to(self.device),
|
| 44 |
+
text_conditioner=AnimaTextConditioner.from_pretrained(
|
| 45 |
+
path, subfolder="text_conditioner", torch_dtype=dtype, **kw
|
| 46 |
+
).eval().requires_grad_(False).to(self.device),
|
| 47 |
+
)
|
| 48 |
+
|
| 49 |
+
@torch.no_grad()
|
| 50 |
+
def __call__(self, prompts):
|
| 51 |
+
from diffusers.modular_pipelines.anima.encoders import AnimaTextEncoderStep
|
| 52 |
+
|
| 53 |
+
c = self.components
|
| 54 |
+
q = AnimaTextEncoderStep.encode_prompt(
|
| 55 |
+
c, prompts, prepare_unconditional_embeds=False, max_sequence_length=self.max_length,
|
| 56 |
+
device=self.device, dtype=self.dtype,
|
| 57 |
+
)
|
| 58 |
+
out = c.text_conditioner(
|
| 59 |
+
source_hidden_states=q["qwen_prompt_embeds"], source_attention_mask=q["qwen_attention_mask"],
|
| 60 |
+
target_input_ids=q["t5_input_ids"], target_attention_mask=q["t5_attention_mask"],
|
| 61 |
+
)
|
| 62 |
+
if not torch.isfinite(out).all():
|
| 63 |
+
raise FloatingPointError("Non-finite prompt embedding")
|
| 64 |
+
return out
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
class VAE:
|
| 68 |
+
"""Qwen-Image VAE: PIL ↔ normalised latent (B, 16, 1, H/8, W/8), (z − mean) / std."""
|
| 69 |
+
|
| 70 |
+
def __init__(self, path=MODEL, device="cuda", dtype=torch.bfloat16):
|
| 71 |
+
from diffusers import AutoencoderKLQwenImage
|
| 72 |
+
|
| 73 |
+
self.vae = AutoencoderKLQwenImage.from_pretrained(
|
| 74 |
+
path, subfolder="vae", torch_dtype=dtype, local_files_only=True
|
| 75 |
+
).to(device).eval().requires_grad_(False)
|
| 76 |
+
self.vae.enable_tiling()
|
| 77 |
+
c = self.vae.config
|
| 78 |
+
self.mean = torch.tensor(c.latents_mean, device=device).view(1, 16, 1, 1, 1)
|
| 79 |
+
self.std = torch.tensor(c.latents_std, device=device).view(1, 16, 1, 1, 1)
|
| 80 |
+
self.device, self.dtype = torch.device(device), dtype
|
| 81 |
+
|
| 82 |
+
@torch.no_grad()
|
| 83 |
+
def encode(self, image):
|
| 84 |
+
px = torch.from_numpy(np.asarray(image.convert("RGB"), dtype=np.float32) / 127.5 - 1)
|
| 85 |
+
px = px.permute(2, 0, 1)[None, :, None].to(self.device, self.dtype)
|
| 86 |
+
z = self.vae.encode(px).latent_dist.mode()
|
| 87 |
+
return ((z.float() - self.mean) / self.std)
|
| 88 |
+
|
| 89 |
+
@torch.no_grad()
|
| 90 |
+
def decode(self, latent):
|
| 91 |
+
"""Decode a latent to an RGB image with round-to-nearest pixel conversion."""
|
| 92 |
+
px = self.vae.decode((latent * self.std + self.mean).to(self.vae.dtype), return_dict=False)[0][0, :, 0]
|
| 93 |
+
px = px.float().div(2).add(0.5).clamp(0, 1)
|
| 94 |
+
return Image.fromarray((px.permute(1, 2, 0).cpu().numpy() * 255).round().astype(np.uint8))
|
anima_role_ref_kv/lora.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""LoRA module utilities for Anima Role Ref KV."""
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
from torch import nn
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class LoRALinear(nn.Module):
|
| 8 |
+
def __init__(self, base: nn.Linear, rank, alpha):
|
| 9 |
+
super().__init__()
|
| 10 |
+
object.__setattr__(self, "base", base) # frozen; not part of this module's parameters
|
| 11 |
+
self.down = nn.Parameter(torch.empty(rank, base.in_features, device=base.weight.device))
|
| 12 |
+
self.up = nn.Parameter(torch.empty(base.out_features, rank, device=base.weight.device))
|
| 13 |
+
self.scale = alpha / rank
|
| 14 |
+
self.multiplier = 1.0
|
| 15 |
+
|
| 16 |
+
def forward(self, x):
|
| 17 |
+
out = self.base(x)
|
| 18 |
+
if self.multiplier == 0:
|
| 19 |
+
return out
|
| 20 |
+
lora = (x.to(self.down.dtype) @ self.down.t()) @ self.up.t()
|
| 21 |
+
return out + (self.multiplier * self.scale) * lora.to(out.dtype)
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def _get(root, dotted):
|
| 25 |
+
m = root
|
| 26 |
+
for part in dotted.split("."):
|
| 27 |
+
m = m[int(part)] if part.isdigit() else getattr(m, part)
|
| 28 |
+
return m
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def _set(root, dotted, value):
|
| 32 |
+
parent, _, name = dotted.rpartition(".")
|
| 33 |
+
p = _get(root, parent) if parent else root
|
| 34 |
+
if name.isdigit():
|
| 35 |
+
p[int(name)] = value
|
| 36 |
+
else:
|
| 37 |
+
setattr(p, name, value)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def inject(net, rank, alpha, targets):
|
| 41 |
+
"""Replace the target linears with LoRALinear; returns {name: LoRALinear}."""
|
| 42 |
+
out = {}
|
| 43 |
+
for i, block in enumerate(net.transformer_blocks):
|
| 44 |
+
for t in targets:
|
| 45 |
+
base = _get(block, t)
|
| 46 |
+
if not isinstance(base, nn.Linear):
|
| 47 |
+
raise TypeError(f"block {i} {t} is {type(base).__name__}")
|
| 48 |
+
lora = LoRALinear(base, rank, alpha)
|
| 49 |
+
_set(block, t, lora)
|
| 50 |
+
out[f"blocks.{i}.{t}"] = lora
|
| 51 |
+
return out
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
class LoRA(nn.Module):
|
| 55 |
+
"""Container for the loaded inference adapter."""
|
| 56 |
+
|
| 57 |
+
def __init__(self, net, rank, alpha, targets):
|
| 58 |
+
super().__init__()
|
| 59 |
+
self.rank, self.alpha = rank, alpha
|
| 60 |
+
self.layers = nn.ModuleDict({k.replace(".", "__"): v for k, v in inject(net, rank, alpha, targets).items()})
|
anima_role_ref_kv/pipeline.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Reference inference on native Diffusers components; plain CFG, reference on positive only."""
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
import torch
|
| 4 |
+
from PIL import Image, ImageOps
|
| 5 |
+
from . import reference as reference_ops
|
| 6 |
+
from .components import NEGATIVE, PromptEncoder, VAE, load_transformer, schedule
|
| 7 |
+
from .weights import load
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def resize_reference(image, area=768 * 768):
|
| 11 |
+
image = ImageOps.exif_transpose(image).convert("RGB")
|
| 12 |
+
s = min(1., (area / (image.width * image.height)) ** .5)
|
| 13 |
+
size = tuple(max(16, int(n * s) // 16 * 16) for n in image.size)
|
| 14 |
+
return image if image.size == size else image.resize(size, Image.Resampling.LANCZOS)
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
@torch.no_grad()
|
| 18 |
+
def sample(net, positive, negative, reference, *, seed=0, width=832, height=1216, steps=28, cfg=4., shift=3., progress_callback=None):
|
| 19 |
+
if width % 16 or height % 16 or min(width, height, steps) <= 0:
|
| 20 |
+
raise ValueError("Positive dimensions must be multiples of 16; steps must be positive")
|
| 21 |
+
device = positive.device
|
| 22 |
+
kv = None
|
| 23 |
+
with torch.autocast(device.type, dtype=torch.bfloat16, enabled=net.dtype == torch.bfloat16):
|
| 24 |
+
if reference is not None:
|
| 25 |
+
kv = reference_ops.encode_reference(net, reference.to(net.dtype), positive, (height // 8, width // 8))
|
| 26 |
+
x = torch.randn(1, 16, 1, height // 8, width // 8,
|
| 27 |
+
generator=torch.Generator(device="cpu").manual_seed(seed)).to(device)
|
| 28 |
+
sigmas = schedule(steps, shift, device)
|
| 29 |
+
for i, (a, b) in enumerate(zip(sigmas[:-1], sigmas[1:])):
|
| 30 |
+
un = reference_ops.velocity(net, x.to(net.dtype), a[None], negative) if cfg != 1 else None
|
| 31 |
+
cond = reference_ops.velocity(net, x.to(net.dtype), a[None], positive, kv)
|
| 32 |
+
velocity = cond if cfg == 1 else un + cfg * (cond - un)
|
| 33 |
+
x = x + (b - a) * velocity.float()
|
| 34 |
+
if progress_callback is not None:
|
| 35 |
+
progress_callback(i + 1, steps)
|
| 36 |
+
return x
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
class AnimaRoleRefKVPipeline:
|
| 40 |
+
def __init__(self, base_model, adapter, device="cuda", format="diffusers"):
|
| 41 |
+
self.net = load_transformer(base_model, device)
|
| 42 |
+
self.adapter = load(self.net, adapter, format)
|
| 43 |
+
self.text = PromptEncoder(base_model, device)
|
| 44 |
+
self.vae = VAE(base_model, device)
|
| 45 |
+
|
| 46 |
+
@torch.no_grad()
|
| 47 |
+
def __call__(self, prompt, reference_image, negative_prompt=NEGATIVE, **kwargs):
|
| 48 |
+
if isinstance(reference_image, (str, Path)):
|
| 49 |
+
reference_image = Image.open(reference_image)
|
| 50 |
+
ref = self.vae.encode(resize_reference(reference_image)) if reference_image is not None else None
|
| 51 |
+
return self.vae.decode(sample(self.net, self.text([prompt]), self.text([negative_prompt]), ref, **kwargs))
|
anima_role_ref_kv/reference.py
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Reference inference functions for Anima Role Ref KV.
|
| 2 |
+
|
| 3 |
+
Portions adapted from Hugging Face Diffusers transformer_cosmos.py (Apache-2.0).
|
| 4 |
+
See NOTICE.txt for attribution and license information."""
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
from diffusers.models.attention_dispatch import dispatch_attention_fn
|
| 8 |
+
from diffusers.models.embeddings import apply_rotary_emb
|
| 9 |
+
|
| 10 |
+
REF_FRAME = 10
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def _tokens(net, latent):
|
| 14 |
+
"""(B, 16, T, h, w) latent → (input with Anima's all-zero padding-mask channel, patch tokens
|
| 15 |
+
(B, N, D), post-patch grid (h', w'))."""
|
| 16 |
+
b, _, t, h, w = latent.shape
|
| 17 |
+
x = torch.cat([latent, latent.new_zeros(b, 1, t, h, w)], dim=1)
|
| 18 |
+
_, p_h, p_w = net.config.patch_size
|
| 19 |
+
return x, net.patch_embed(x).flatten(1, 3), (h // p_h, w // p_w)
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def rope_at(net, frame, hs, ws):
|
| 23 |
+
"""Anima's RoPE (CosmosRotaryPosEmbed, images) at frame index `frame` and integer h/w
|
| 24 |
+
positions `hs`, `ws` (1-D, may be negative or beyond the table) → (cos, sin), (len(hs)·len(ws), head_dim)."""
|
| 25 |
+
r = net.rope
|
| 26 |
+
device = hs.device
|
| 27 |
+
h_theta, w_theta, t_theta = 10000.0 * r.h_ntk_factor, 10000.0 * r.w_ntk_factor, 10000.0 * r.t_ntk_factor
|
| 28 |
+
|
| 29 |
+
def ranges(d):
|
| 30 |
+
return torch.arange(0, d, 2, device=device, dtype=torch.float32)[: (d // 2)] / d
|
| 31 |
+
|
| 32 |
+
hf, wf, tf = 1.0 / (h_theta ** ranges(r.dim_h)), 1.0 / (w_theta ** ranges(r.dim_w)), 1.0 / (t_theta ** ranges(r.dim_t))
|
| 33 |
+
nh, nw = len(hs), len(ws)
|
| 34 |
+
et = torch.outer(torch.tensor([float(frame)], device=device), tf)[:, None, None, :].repeat(1, nh, nw, 1)
|
| 35 |
+
eh = torch.outer(hs.float(), hf)[None, :, None, :].repeat(1, 1, nw, 1)
|
| 36 |
+
ew = torch.outer(ws.float(), wf)[None, None, :, :].repeat(1, nh, 1, 1)
|
| 37 |
+
freqs = torch.cat([et, eh, ew] * 2, dim=-1).flatten(0, 2).float()
|
| 38 |
+
return torch.cos(freqs), torch.sin(freqs)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def ref_positions(target_hw, ref_hw, device):
|
| 42 |
+
"""Centre the reference grid on the target grid."""
|
| 43 |
+
(ht, wt), (hr, wr) = target_hw, ref_hw
|
| 44 |
+
hs = torch.arange(hr, device=device) + (ht // 2 - hr // 2)
|
| 45 |
+
ws = torch.arange(wr, device=device) + (wt // 2 - wr // 2)
|
| 46 |
+
return hs, ws
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def _project(attn, x):
|
| 50 |
+
"""q, k (after QK-norm, before RoPE), v: (B, heads, N, head_dim)."""
|
| 51 |
+
q, k, v = (f(x).unflatten(2, (attn.heads, -1)).transpose(1, 2) for f in (attn.to_q, attn.to_k, attn.to_v))
|
| 52 |
+
return attn.norm_q(q), attn.norm_k(k), v
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def _rotate(t, rope):
|
| 56 |
+
return apply_rotary_emb(t, rope, use_real=True, use_real_unbind_dim=-2)
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def _qkv(attn, x, rope):
|
| 60 |
+
q, k, v = _project(attn, x)
|
| 61 |
+
return _rotate(q, rope), _rotate(k, rope), v
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def _softmax_attention(q, k, v):
|
| 65 |
+
"""(B, heads, N, d) → (B, N, heads·d)."""
|
| 66 |
+
out = dispatch_attention_fn(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2),
|
| 67 |
+
attn_mask=None, dropout_p=0.0, is_causal=False)
|
| 68 |
+
return out.flatten(2, 3).type_as(q)
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def _attend(attn, q, k, v):
|
| 72 |
+
return attn.to_out[1](attn.to_out[0](_softmax_attention(q, k, v)))
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def _ref_block(block, h, ctx, emb, temb, rope, last):
|
| 76 |
+
n, gate = block.norm1(h, emb, temb)
|
| 77 |
+
qp, kp, v = _project(block.attn1, n)
|
| 78 |
+
q, k = _rotate(qp, rope), _rotate(kp, rope)
|
| 79 |
+
if last:
|
| 80 |
+
return h, k, v
|
| 81 |
+
h = h + gate * _attend(block.attn1, q, k, v)
|
| 82 |
+
n, gate = block.norm2(h, emb, temb)
|
| 83 |
+
h = h + gate * block.attn2(n, encoder_hidden_states=ctx, attention_mask=None)
|
| 84 |
+
n, gate = block.norm3(h, emb, temb)
|
| 85 |
+
return h + gate * block.ff(n), k, v
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def _target_block(block, h, ctx, emb, temb, rope, k_ref, v_ref):
|
| 89 |
+
n, gate = block.norm1(h, emb, temb)
|
| 90 |
+
q, k, v = _qkv(block.attn1, n, rope)
|
| 91 |
+
if k_ref is not None:
|
| 92 |
+
b = k.shape[0]
|
| 93 |
+
k = torch.cat([k_ref.expand(b, -1, -1, -1).to(k.dtype), k], dim=2)
|
| 94 |
+
v = torch.cat([v_ref.expand(b, -1, -1, -1).to(v.dtype), v], dim=2)
|
| 95 |
+
out = _attend(block.attn1, q, k, v)
|
| 96 |
+
h = h + gate * out
|
| 97 |
+
n, gate = block.norm2(h, emb, temb)
|
| 98 |
+
h = h + gate * block.attn2(n, encoder_hidden_states=ctx, attention_mask=None)
|
| 99 |
+
n, gate = block.norm3(h, emb, temb)
|
| 100 |
+
return h + gate * block.ff(n)
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def encode_reference(net, ref, ctx, target_hw):
|
| 104 |
+
"""Encode a reference latent into the per-block key/value cache."""
|
| 105 |
+
_, p_h, p_w = net.config.patch_size
|
| 106 |
+
_, h, ref_hw = _tokens(net, ref.to(net.dtype))
|
| 107 |
+
hs, ws = ref_positions((target_hw[0] // p_h, target_hw[1] // p_w), ref_hw, h.device)
|
| 108 |
+
rope = rope_at(net, REF_FRAME, hs, ws)
|
| 109 |
+
temb, emb = net.time_embed(h, torch.zeros(h.shape[0], device=h.device))
|
| 110 |
+
blocks = net.transformer_blocks
|
| 111 |
+
kv = []
|
| 112 |
+
for i, block in enumerate(blocks):
|
| 113 |
+
h, k, v = _ref_block(block, h, ctx, emb, temb, rope, i == len(blocks) - 1)
|
| 114 |
+
kv.append((k, v))
|
| 115 |
+
return kv
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def velocity(net, x, sigma, ctx, ref_kv=None):
|
| 119 |
+
"""Predict a float32 flow velocity, optionally conditioned on a reference."""
|
| 120 |
+
xin, h, (hp, wp) = _tokens(net, x)
|
| 121 |
+
rope = net.rope(xin, fps=None)
|
| 122 |
+
temb, emb = net.time_embed(h, sigma)
|
| 123 |
+
for i, block in enumerate(net.transformer_blocks):
|
| 124 |
+
k_ref, v_ref = ref_kv[i] if ref_kv is not None else (None, None)
|
| 125 |
+
h = _target_block(block, h, ctx, emb, temb, rope, k_ref, v_ref)
|
| 126 |
+
h = net.proj_out(net.norm_out(h, emb, temb))
|
| 127 |
+
p_t, p_h, p_w = net.config.patch_size
|
| 128 |
+
h = h.unflatten(2, (p_h, p_w, p_t, -1)).unflatten(1, (x.shape[2] // p_t, hp, wp))
|
| 129 |
+
return h.permute(0, 7, 1, 6, 2, 4, 3, 5).flatten(6, 7).flatten(4, 5).flatten(2, 3).float()
|
anima_role_ref_kv/weights.py
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Strict loading of the published Diffusers and ComfyUI adapter formats."""
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
from safetensors.torch import load_file
|
| 5 |
+
|
| 6 |
+
from .lora import LoRA
|
| 7 |
+
|
| 8 |
+
TARGETS = {
|
| 9 |
+
"attn1.to_q": "self_attn.q_proj", "attn1.to_k": "self_attn.k_proj",
|
| 10 |
+
"attn1.to_v": "self_attn.v_proj", "attn1.to_out.0": "self_attn.output_proj",
|
| 11 |
+
"ff.net.0.proj": "mlp.layer1", "ff.net.2": "mlp.layer2",
|
| 12 |
+
}
|
| 13 |
+
BLOCKS, RANK, ALPHA = 28, 128, 128.0
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def _weight_key(block, target, side, format):
|
| 17 |
+
if format == "diffusers":
|
| 18 |
+
suffix = "A" if side == "down" else "B"
|
| 19 |
+
return f"transformer.transformer_blocks.{block}.{target}.lora_{suffix}.weight"
|
| 20 |
+
return f"diffusion_model.blocks.{block}.{TARGETS[target]}.lora_{side}.weight"
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def load(net, filename, format="diffusers"):
|
| 24 |
+
"""Validate the complete adapter before attaching it to an Anima base model."""
|
| 25 |
+
if format not in ("diffusers", "comfy"):
|
| 26 |
+
raise ValueError(f"Unsupported adapter format: {format}")
|
| 27 |
+
if len(net.transformer_blocks) != BLOCKS:
|
| 28 |
+
raise ValueError(f"Expected {BLOCKS} Anima transformer blocks")
|
| 29 |
+
source = load_file(str(filename))
|
| 30 |
+
state, expected = {}, set()
|
| 31 |
+
for block in range(BLOCKS):
|
| 32 |
+
for target in TARGETS:
|
| 33 |
+
module = net.transformer_blocks[block]
|
| 34 |
+
for part in target.split("."):
|
| 35 |
+
module = module[int(part)] if part.isdigit() else getattr(module, part)
|
| 36 |
+
for side, shape in (("down", (RANK, module.in_features)),
|
| 37 |
+
("up", (module.out_features, RANK))):
|
| 38 |
+
key = _weight_key(block, target, side, format)
|
| 39 |
+
expected.add(key)
|
| 40 |
+
value = source[key]
|
| 41 |
+
if tuple(value.shape) != shape:
|
| 42 |
+
raise ValueError(f"Adapter shape mismatch: {key}; expected {shape}")
|
| 43 |
+
if value.dtype != torch.float32 or not torch.isfinite(value).all():
|
| 44 |
+
raise ValueError(f"Expected finite FP32 adapter weights: {key}")
|
| 45 |
+
state[f"layers.blocks__{block}__{target.replace('.', '__')}.{side}"] = value
|
| 46 |
+
if format == "comfy":
|
| 47 |
+
key = f"diffusion_model.blocks.{block}.{TARGETS[target]}.alpha"
|
| 48 |
+
expected.add(key)
|
| 49 |
+
if source[key].numel() != 1 or float(source[key]) != ALPHA:
|
| 50 |
+
raise ValueError(f"Unsupported adapter alpha: {key}")
|
| 51 |
+
if set(source) != expected:
|
| 52 |
+
raise ValueError(f"Unexpected adapter keys: {sorted(set(source) - expected)}")
|
| 53 |
+
adapter = LoRA(net, RANK, ALPHA, tuple(TARGETS))
|
| 54 |
+
adapter.load_state_dict(state, strict=True)
|
| 55 |
+
return adapter.eval().requires_grad_(False)
|
app.py
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Character-reference demo for six official Anima checkpoints on ZeroGPU."""
|
| 2 |
+
import os
|
| 3 |
+
import random
|
| 4 |
+
import time
|
| 5 |
+
|
| 6 |
+
import spaces # Import before torch and all modules that import torch.
|
| 7 |
+
import gradio as gr
|
| 8 |
+
from huggingface_hub import hf_hub_download, snapshot_download
|
| 9 |
+
from PIL import ImageOps
|
| 10 |
+
|
| 11 |
+
from models import (MODELS, DEFAULT_MODEL, RESOLUTIONS, BASE_REPO, BASE_REVISION,
|
| 12 |
+
MODEL_REPO, MODEL_REVISION, ADAPTER_REPO, ADAPTER_REVISION,
|
| 13 |
+
ADAPTER_FILE, model_settings)
|
| 14 |
+
from anima_role_ref_kv.components import NEGATIVE
|
| 15 |
+
from anima_role_ref_kv.pipeline import resize_reference
|
| 16 |
+
from runtime import Runtime
|
| 17 |
+
|
| 18 |
+
BASE = os.environ.get("ANIMA_BASE_DIR") or snapshot_download(
|
| 19 |
+
BASE_REPO, revision=BASE_REVISION,
|
| 20 |
+
allow_patterns=["*.json", "transformer/*", "text_conditioner/*", "text_encoder/*",
|
| 21 |
+
"tokenizer/*", "t5_tokenizer/*", "vae/*", "scheduler/*"])
|
| 22 |
+
ADAPTER = os.environ.get("ANIMA_ADAPTER_FILE") or hf_hub_download(
|
| 23 |
+
ADAPTER_REPO, ADAPTER_FILE, revision=ADAPTER_REVISION)
|
| 24 |
+
# ZeroGPU captures CUDA placement at startup. Only one backbone, text stack and
|
| 25 |
+
# VAE are resident. Official variants are fetched on CPU before requesting GPU.
|
| 26 |
+
PIPE = Runtime(BASE, ADAPTER)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def gpu_duration(model_key, filename, reference, prompt, negative, width, height,
|
| 30 |
+
steps, cfg, strength, seed, progress):
|
| 31 |
+
return min(120, int(20 + steps * (0.45 if cfg == 1 else 0.85)))
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
@spaces.GPU(duration=gpu_duration)
|
| 35 |
+
def gpu_generate(model_key, filename, reference, prompt, negative, width, height,
|
| 36 |
+
steps, cfg, strength, seed, progress):
|
| 37 |
+
start = time.perf_counter()
|
| 38 |
+
progress(0, desc="Loading selected base and encoding character reference…")
|
| 39 |
+
result = PIPE.generate(
|
| 40 |
+
model_key, filename, reference, prompt, negative_prompt=negative,
|
| 41 |
+
width=width, height=height, steps=steps, cfg=cfg, strength=strength, seed=seed,
|
| 42 |
+
progress_callback=lambda i, n: progress(i / n, desc=f"Denoising {i}/{n}"))
|
| 43 |
+
return result, time.perf_counter() - start
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def generate(reference, prompt, model_key, resolution, negative, steps, cfg,
|
| 47 |
+
strength, seed, randomize, quality_prefix, progress=gr.Progress()):
|
| 48 |
+
# Public API validation and model downloads take place BEFORE GPU allocation.
|
| 49 |
+
if reference is None:
|
| 50 |
+
raise gr.Error("请上传角色参考图 / Upload a character reference image.")
|
| 51 |
+
if model_key not in MODELS or resolution not in RESOLUTIONS:
|
| 52 |
+
raise gr.Error("Unknown model or resolution.")
|
| 53 |
+
prompt = (prompt or "").strip()
|
| 54 |
+
if not prompt or len(prompt) > 4000:
|
| 55 |
+
raise gr.Error("Prompt must contain 1–4000 characters.")
|
| 56 |
+
negative = negative or ""
|
| 57 |
+
if len(negative) > 4000:
|
| 58 |
+
raise gr.Error("Negative prompt must not exceed 4000 characters.")
|
| 59 |
+
try:
|
| 60 |
+
steps, cfg, strength = int(steps), float(cfg), float(strength)
|
| 61 |
+
if not (1 <= steps <= 50 and 1 <= cfg <= 10 and 0 <= strength <= 1.5):
|
| 62 |
+
raise ValueError
|
| 63 |
+
seed = random.randint(0, 2**31 - 1) if randomize else int(seed)
|
| 64 |
+
if not 0 <= seed <= 2**31 - 1:
|
| 65 |
+
raise ValueError
|
| 66 |
+
except (ValueError, TypeError, OverflowError):
|
| 67 |
+
raise gr.Error("Invalid steps, CFG, adapter strength, or seed.")
|
| 68 |
+
reference = resize_reference(ImageOps.exif_transpose(reference).convert("RGB"))
|
| 69 |
+
if quality_prefix:
|
| 70 |
+
prompt = "masterpiece, best quality, " + prompt
|
| 71 |
+
filename = None
|
| 72 |
+
if model_key != DEFAULT_MODEL:
|
| 73 |
+
progress(0, desc="Preparing official checkpoint (first use downloads weights)…")
|
| 74 |
+
filename = hf_hub_download(MODEL_REPO,
|
| 75 |
+
"split_files/diffusion_models/" + MODELS[model_key].filename,
|
| 76 |
+
revision=MODEL_REVISION)
|
| 77 |
+
width, height = RESOLUTIONS[resolution]
|
| 78 |
+
image, seconds = gpu_generate(model_key, filename, reference, prompt, negative,
|
| 79 |
+
width, height, steps, cfg, strength, seed, progress)
|
| 80 |
+
metadata = {"model": model_key, "base_repo": BASE_REPO if filename is None else MODEL_REPO,
|
| 81 |
+
"base_revision": BASE_REVISION if filename is None else MODEL_REVISION,
|
| 82 |
+
"adapter": ADAPTER_REPO, "adapter_revision": ADAPTER_REVISION,
|
| 83 |
+
"prompt": prompt, "negative_prompt": negative, "negative_active": cfg != 1,
|
| 84 |
+
"seed": seed, "width": width, "height": height,
|
| 85 |
+
"steps": steps, "cfg": cfg, "adapter_strength": strength,
|
| 86 |
+
"scheduler": "flow-euler", "shift": 3.0, "seconds": round(seconds, 2)}
|
| 87 |
+
return image, seed, f"{MODELS[model_key].label} · {steps} steps · CFG {cfg:g} · {seconds:.1f}s", metadata
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def update_model(key):
|
| 91 |
+
steps, cfg, note = model_settings(key)
|
| 92 |
+
negative = "worst quality, low quality, blurry, jpeg artifacts" if key.startswith("aesthetic") else NEGATIVE
|
| 93 |
+
return steps, cfg, note, gr.update(value=negative, interactive=cfg != 1)
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
CSS = """
|
| 97 |
+
.gradio-container {max-width: 1180px !important; margin: auto;}
|
| 98 |
+
#hero {padding: 24px 0 12px;} #generate {min-height: 52px;}
|
| 99 |
+
"""
|
| 100 |
+
with gr.Blocks(title="Anima Role Ref KV · Official Bases", analytics_enabled=False) as demo:
|
| 101 |
+
gr.Markdown("""# Anima Role Ref KV
|
| 102 |
+
### 一张角色参考图,探索六款 Anima 官方底模
|
| 103 |
+
Upload a character reference, choose a base, and describe a new pose, outfit or scene.
|
| 104 |
+
The Role Ref KV adapter is applied automatically. No per-character training required.
|
| 105 |
+
""", elem_id="hero")
|
| 106 |
+
with gr.Row():
|
| 107 |
+
with gr.Column():
|
| 108 |
+
reference = gr.Image(label="角色参考图 / Character reference", type="pil", height=340,
|
| 109 |
+
sources=["upload", "clipboard"])
|
| 110 |
+
model = gr.Dropdown(choices=[(m.label, k) for k, m in MODELS.items()],
|
| 111 |
+
value=DEFAULT_MODEL, label="官方底模 / Official base model")
|
| 112 |
+
note = gr.Markdown(model_settings(DEFAULT_MODEL)[2])
|
| 113 |
+
prompt = gr.Textbox(label="提示词 / Prompt", lines=3,
|
| 114 |
+
value="1girl, solo, upper body, smile, looking at viewer, garden, sunlight")
|
| 115 |
+
run = gr.Button("生成 / Generate", variant="primary", elem_id="generate")
|
| 116 |
+
with gr.Column():
|
| 117 |
+
result = gr.Image(label="生成结果 / Result", height=520, format="png")
|
| 118 |
+
summary = gr.Textbox(label="Generation", interactive=False)
|
| 119 |
+
used_seed = gr.Number(label="Seed used", interactive=False, precision=0)
|
| 120 |
+
with gr.Accordion("生成设置 / Generation settings", open=True):
|
| 121 |
+
resolution = gr.Radio(list(RESOLUTIONS), value=next(iter(RESOLUTIONS)), label="Resolution")
|
| 122 |
+
with gr.Row():
|
| 123 |
+
steps = gr.Slider(1, 50, value=28, step=1, label="Steps")
|
| 124 |
+
cfg = gr.Slider(1, 10, value=4, step=.5, label="CFG")
|
| 125 |
+
strength = gr.Slider(0, 1.5, value=1, step=.05, label="Role Ref KV strength",
|
| 126 |
+
info="1.0 recommended. 0 disables the LoRA delta; reference KV remains active.")
|
| 127 |
+
negative = gr.Textbox(value=NEGATIVE, label="Negative prompt (inactive at CFG 1)", lines=2)
|
| 128 |
+
with gr.Row():
|
| 129 |
+
seed = gr.Number(value=0, precision=0, minimum=0, maximum=2**31-1, label="Seed")
|
| 130 |
+
randomize = gr.Checkbox(value=True, label="Randomize seed")
|
| 131 |
+
prefix = gr.Checkbox(value=True, label="Add masterpiece, best quality")
|
| 132 |
+
with gr.Accordion("复现参数 / Reproducibility", open=False):
|
| 133 |
+
metadata = gr.JSON(label="Generation metadata")
|
| 134 |
+
gr.Markdown("""Use a clear single-character reference. Selecting a model resets its suggested steps and CFG.
|
| 135 |
+
Turbo defaults to **8 steps / CFG 1**; try 8–12 steps. First use of each base downloads its weights.
|
| 136 |
+
Only Base v1.0 is validated by the adapter release; Aesthetic and Turbo combinations are experimental.
|
| 137 |
+
|
| 138 |
+
[Model & code](https://huggingface.co/AnticPan/Anima-Role-Ref-KV) ·
|
| 139 |
+
[Official Anima checkpoints](https://huggingface.co/circlestone-labs/Anima) ·
|
| 140 |
+
[Original demo by Hugging Face](https://huggingface.co/spaces/hugging-apps/anima-role-ref-kv-2)
|
| 141 |
+
|
| 142 |
+
Independent Anima derivative, not endorsed by CircleStone Labs. Model weights are subject to the
|
| 143 |
+
[CircleStone Labs Non-Commercial License](https://huggingface.co/circlestone-labs/Anima/blob/main/LICENSE.md).
|
| 144 |
+
Runtime code: Apache-2.0; see NOTICE.txt. Thanks to Apolinario and the Hugging Face team for the original demo.
|
| 145 |
+
""")
|
| 146 |
+
model.change(update_model, model, [steps, cfg, note, negative], queue=False)
|
| 147 |
+
cfg.change(lambda v: gr.update(interactive=float(v) != 1), cfg, negative, queue=False)
|
| 148 |
+
run.click(generate, [reference, prompt, model, resolution, negative, steps, cfg, strength,
|
| 149 |
+
seed, randomize, prefix], [result, used_seed, summary, metadata],
|
| 150 |
+
api_name="generate", concurrency_limit=1, concurrency_id="anima")
|
| 151 |
+
|
| 152 |
+
if __name__ == "__main__":
|
| 153 |
+
demo.queue(max_size=16, default_concurrency_limit=1).launch(
|
| 154 |
+
theme=gr.themes.Soft(), css=CSS, show_error=True, ssr_mode=False)
|
checkpoints.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Strict, lossless single-file Anima key mapping and in-place weight switching.
|
| 2 |
+
|
| 3 |
+
Mapping follows the workspace's anima-convert/convert.py and Diffusers Cosmos
|
| 4 |
+
converter. Text conditioner weights belong to the chosen checkpoint, too.
|
| 5 |
+
"""
|
| 6 |
+
import re
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
|
| 9 |
+
import torch
|
| 10 |
+
from safetensors import safe_open
|
| 11 |
+
|
| 12 |
+
TOP = {
|
| 13 |
+
"x_embedder.proj.1.weight": "patch_embed.proj.weight",
|
| 14 |
+
"t_embedder.1.linear_1.weight": "time_embed.t_embedder.linear_1.weight",
|
| 15 |
+
"t_embedder.1.linear_2.weight": "time_embed.t_embedder.linear_2.weight",
|
| 16 |
+
"t_embedding_norm.weight": "time_embed.norm.weight",
|
| 17 |
+
"final_layer.adaln_modulation.1.weight": "norm_out.linear_1.weight",
|
| 18 |
+
"final_layer.adaln_modulation.2.weight": "norm_out.linear_2.weight",
|
| 19 |
+
"final_layer.linear.weight": "proj_out.weight",
|
| 20 |
+
}
|
| 21 |
+
BLOCK = {
|
| 22 |
+
**{f"{a}.{old}.weight": f"{b}.{new}.weight"
|
| 23 |
+
for a, b in [("self_attn", "attn1"), ("cross_attn", "attn2")]
|
| 24 |
+
for old, new in [("q_proj", "to_q"), ("k_proj", "to_k"), ("v_proj", "to_v"),
|
| 25 |
+
("output_proj", "to_out.0"), ("q_norm", "norm_q"), ("k_norm", "norm_k")]},
|
| 26 |
+
**{f"adaln_modulation_{a}.{i}.weight": f"norm{n}.linear_{i}.weight"
|
| 27 |
+
for a, n in [("self_attn", 1), ("cross_attn", 2), ("mlp", 3)] for i in (1, 2)},
|
| 28 |
+
"mlp.layer1.weight": "ff.net.0.proj.weight",
|
| 29 |
+
"mlp.layer2.weight": "ff.net.2.weight",
|
| 30 |
+
}
|
| 31 |
+
BUFFERS = {"pos_embedder.seq", "pos_embedder.dim_spatial_range", "pos_embedder.dim_temporal_range"}
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def map_key(key):
|
| 35 |
+
if key in TOP:
|
| 36 |
+
return "transformer", TOP[key]
|
| 37 |
+
if key.startswith("llm_adapter."):
|
| 38 |
+
return "conditioner", key.removeprefix("llm_adapter.")
|
| 39 |
+
if key in BUFFERS:
|
| 40 |
+
return None
|
| 41 |
+
match = re.fullmatch(r"blocks\.(\d+)\.(.+)", key)
|
| 42 |
+
if match and match[2] in BLOCK:
|
| 43 |
+
return "transformer", f"transformer_blocks.{match[1]}.{BLOCK[match[2]]}"
|
| 44 |
+
raise ValueError(f"Unsupported checkpoint tensor: {key}")
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def native_plan(filename, targets):
|
| 48 |
+
"""Check every key and shape before touching a live model."""
|
| 49 |
+
plan, seen = [], set()
|
| 50 |
+
with safe_open(filename, framework="pt", device="cpu") as source:
|
| 51 |
+
keys = list(source.keys())
|
| 52 |
+
prefix = next((p for p in ("model.diffusion_model.", "diffusion_model.", "net.")
|
| 53 |
+
if keys and all(k.startswith(p) for k in keys)), "")
|
| 54 |
+
for key in keys:
|
| 55 |
+
mapped = map_key(key[len(prefix):])
|
| 56 |
+
if mapped is None:
|
| 57 |
+
continue
|
| 58 |
+
if mapped not in targets or mapped in seen:
|
| 59 |
+
raise ValueError(f"Unexpected or duplicate tensor: {key}")
|
| 60 |
+
tensor = source.get_slice(key)
|
| 61 |
+
if tuple(tensor.get_shape()) != tuple(targets[mapped].shape):
|
| 62 |
+
raise ValueError(f"Shape mismatch: {key}")
|
| 63 |
+
if tensor.get_dtype() not in {"BF16", "F16", "F32"}:
|
| 64 |
+
raise ValueError(f"Unsupported weight dtype: {key}")
|
| 65 |
+
seen.add(mapped)
|
| 66 |
+
plan.append((str(filename), key, mapped))
|
| 67 |
+
if seen != set(targets):
|
| 68 |
+
raise ValueError(f"Missing checkpoint tensors: {sorted(set(targets) - seen)}")
|
| 69 |
+
return plan
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def diffusers_plan(directory, targets):
|
| 73 |
+
import json
|
| 74 |
+
plan, seen = [], set()
|
| 75 |
+
for component, folder in [("transformer", "transformer"), ("conditioner", "text_conditioner")]:
|
| 76 |
+
root = Path(directory) / folder
|
| 77 |
+
index = root / "diffusion_pytorch_model.safetensors.index.json"
|
| 78 |
+
files = (sorted(set(json.loads(index.read_text())["weight_map"].values())) if index.exists()
|
| 79 |
+
else ["diffusion_pytorch_model.safetensors"])
|
| 80 |
+
for name in files:
|
| 81 |
+
with safe_open(root / name, framework="pt", device="cpu") as source:
|
| 82 |
+
for key in source.keys():
|
| 83 |
+
mapped = component, key
|
| 84 |
+
if mapped not in targets or mapped in seen:
|
| 85 |
+
raise ValueError(f"Unexpected or duplicate tensor: {mapped}")
|
| 86 |
+
tensor = source.get_slice(key)
|
| 87 |
+
if tuple(tensor.get_shape()) != tuple(targets[mapped].shape):
|
| 88 |
+
raise ValueError(f"Shape mismatch: {mapped}")
|
| 89 |
+
seen.add(mapped)
|
| 90 |
+
plan.append((str(root / name), key, mapped))
|
| 91 |
+
if seen != set(targets):
|
| 92 |
+
raise ValueError(f"Missing Diffusers tensors: {sorted(set(targets) - seen)}")
|
| 93 |
+
return plan
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
@torch.no_grad()
|
| 97 |
+
def copy_plan(plan, targets):
|
| 98 |
+
from itertools import groupby
|
| 99 |
+
for filename, rows in groupby(plan, key=lambda row: row[0]):
|
| 100 |
+
with safe_open(filename, framework="pt", device="cpu") as source:
|
| 101 |
+
for _, key, mapped in rows:
|
| 102 |
+
targets[mapped].copy_(source.get_tensor(key))
|
models.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Pinned official Anima releases; no arbitrary model paths from the public API."""
|
| 2 |
+
from dataclasses import dataclass
|
| 3 |
+
|
| 4 |
+
BASE_REPO = "circlestone-labs/Anima-Base-v1.0-Diffusers"
|
| 5 |
+
BASE_REVISION = "073c3a9db359c31ad0e8aa268d15775473c2176c"
|
| 6 |
+
MODEL_REPO = "circlestone-labs/Anima"
|
| 7 |
+
MODEL_REVISION = "f973fc41ec7545364ac9776c2440285f43ff2a30"
|
| 8 |
+
ADAPTER_REPO = "AnticPan/Anima-Role-Ref-KV"
|
| 9 |
+
ADAPTER_REVISION = "259a0dce876ed70fc8835b888139756e23fb8a9f"
|
| 10 |
+
ADAPTER_FILE = "diffusers/pytorch_lora_weights.safetensors"
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
@dataclass(frozen=True)
|
| 14 |
+
class Model:
|
| 15 |
+
label: str
|
| 16 |
+
filename: str
|
| 17 |
+
steps: int = 28
|
| 18 |
+
cfg: float = 4.0
|
| 19 |
+
experimental: bool = True
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
MODELS = {
|
| 23 |
+
"base-v1.0": Model("Base v1.0", "anima-base-v1.0.safetensors", experimental=False),
|
| 24 |
+
"aesthetic-v1.0": Model("Aesthetic v1.0", "anima-aesthetic-v1.0.safetensors"),
|
| 25 |
+
"aesthetic-v1.0b": Model("Aesthetic v1.0b", "anima-aesthetic-v1.0b.safetensors"),
|
| 26 |
+
"aesthetic-v1.1": Model("Aesthetic v1.1", "anima-aesthetic-v1.1.safetensors"),
|
| 27 |
+
"turbo-v1.0": Model("Turbo v1.0 · 8 steps", "anima-turbo-v1.0.safetensors", 8, 1.0),
|
| 28 |
+
"turbo-v1.1": Model("Turbo v1.1 · 8 steps", "anima-turbo-v1.1.safetensors", 8, 1.0),
|
| 29 |
+
}
|
| 30 |
+
DEFAULT_MODEL = "base-v1.0"
|
| 31 |
+
RESOLUTIONS = {"Portrait · 832 × 1216": (832, 1216),
|
| 32 |
+
"Square · 1024 × 1024": (1024, 1024),
|
| 33 |
+
"Landscape · 1216 × 832": (1216, 832)}
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def model_settings(key):
|
| 37 |
+
model = MODELS[key]
|
| 38 |
+
note = ("Base v1.0 is the adapter's validated base."
|
| 39 |
+
if not model.experimental else
|
| 40 |
+
"Experimental adapter combination: inference compatibility does not guarantee character fidelity.")
|
| 41 |
+
if model.cfg == 1:
|
| 42 |
+
note += " Turbo: CFG 1, 8–12 steps. Negative prompt is inactive at CFG 1."
|
| 43 |
+
elif key.startswith("aesthetic"):
|
| 44 |
+
note += " Aesthetic: avoid score_* tags."
|
| 45 |
+
return model.steps, model.cfg, note
|
requirements.txt
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
torch>=2.6
|
| 2 |
+
diffusers @ git+https://github.com/huggingface/diffusers.git@5ff8e59ff9fe81c6e2df4fb4c6ea0d97a5df5ab2
|
| 3 |
+
transformers==5.17.0
|
| 4 |
+
accelerate>=1
|
| 5 |
+
safetensors>=0.5
|
| 6 |
+
sentencepiece
|
| 7 |
+
Pillow
|
| 8 |
+
numpy
|
runtime.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""One resident pipeline; original backbone tensors are never cumulatively merged."""
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
import torch
|
| 4 |
+
|
| 5 |
+
from anima_role_ref_kv.components import load_transformer, PromptEncoder, VAE, NEGATIVE
|
| 6 |
+
from anima_role_ref_kv.pipeline import sample, resize_reference
|
| 7 |
+
from anima_role_ref_kv.weights import load
|
| 8 |
+
from checkpoints import native_plan, diffusers_plan, copy_plan
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class Runtime:
|
| 12 |
+
def __init__(self, base_dir, adapter_file, device="cuda"):
|
| 13 |
+
self.base_dir = str(base_dir)
|
| 14 |
+
self.net = load_transformer(base_dir, device)
|
| 15 |
+
self.text = PromptEncoder(base_dir, device)
|
| 16 |
+
self.vae = VAE(base_dir, device)
|
| 17 |
+
# Capture references before LoRALinear wraps linears. The adapter keeps
|
| 18 |
+
# its frozen base outside its registered parameters; net.state_dict()
|
| 19 |
+
# after injection would omit those base weights.
|
| 20 |
+
self.targets = {("transformer", k): v for k, v in self.net.state_dict(keep_vars=True).items()}
|
| 21 |
+
self.targets.update({("conditioner", k): v for k, v in
|
| 22 |
+
self.text.components.text_conditioner.state_dict(keep_vars=True).items()})
|
| 23 |
+
self.base_plan = diffusers_plan(base_dir, self.targets)
|
| 24 |
+
self.adapter = load(self.net, adapter_file)
|
| 25 |
+
self.current = "base-v1.0"
|
| 26 |
+
|
| 27 |
+
def select(self, model_key, filename=None):
|
| 28 |
+
if self.current == model_key:
|
| 29 |
+
return
|
| 30 |
+
plan = (self.base_plan if model_key == "base-v1.0" else
|
| 31 |
+
(diffusers_plan(filename, self.targets) if Path(filename).is_dir() else
|
| 32 |
+
native_plan(filename, self.targets)))
|
| 33 |
+
self.current = None # A failed copy must force a complete reload next time.
|
| 34 |
+
copy_plan(plan, self.targets)
|
| 35 |
+
self.current = model_key
|
| 36 |
+
|
| 37 |
+
@torch.no_grad()
|
| 38 |
+
def generate(self, model_key, filename, reference, prompt, *, strength=1.0,
|
| 39 |
+
negative_prompt=NEGATIVE, progress_callback=None, **settings):
|
| 40 |
+
self.select(model_key, filename)
|
| 41 |
+
for layer in self.adapter.layers.values():
|
| 42 |
+
layer.multiplier = strength
|
| 43 |
+
cfg = settings.get("cfg", 4.)
|
| 44 |
+
positive = self.text([prompt])
|
| 45 |
+
negative = self.text([negative_prompt]) if cfg != 1 else None
|
| 46 |
+
ref = self.vae.encode(resize_reference(reference))
|
| 47 |
+
latent = sample(self.net, positive, negative, ref,
|
| 48 |
+
progress_callback=progress_callback, **settings)
|
| 49 |
+
if not torch.isfinite(latent).all():
|
| 50 |
+
raise FloatingPointError("The model produced non-finite latents")
|
| 51 |
+
return self.vae.decode(latent)
|
smoke.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Bounded GPU inference validation; never trains or writes to model directories."""
|
| 2 |
+
import argparse
|
| 3 |
+
import json
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
import time
|
| 6 |
+
|
| 7 |
+
import numpy as np
|
| 8 |
+
from PIL import Image, ImageOps, ImageDraw
|
| 9 |
+
import torch
|
| 10 |
+
|
| 11 |
+
from models import MODELS
|
| 12 |
+
from runtime import Runtime
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def main():
|
| 16 |
+
p = argparse.ArgumentParser()
|
| 17 |
+
p.add_argument("--base", required=True)
|
| 18 |
+
p.add_argument("--adapter", required=True)
|
| 19 |
+
p.add_argument("--checkpoints", required=True)
|
| 20 |
+
p.add_argument("--reference", required=True)
|
| 21 |
+
p.add_argument("--output", required=True)
|
| 22 |
+
p.add_argument("--width", type=int, default=512)
|
| 23 |
+
p.add_argument("--height", type=int, default=768)
|
| 24 |
+
a = p.parse_args()
|
| 25 |
+
torch.set_num_threads(4)
|
| 26 |
+
out = Path(a.output)
|
| 27 |
+
out.mkdir(parents=True, exist_ok=True)
|
| 28 |
+
ref = Image.open(a.reference).convert("RGB")
|
| 29 |
+
ref.save(out / "reference.png")
|
| 30 |
+
pipe = Runtime(a.base, a.adapter)
|
| 31 |
+
prompt = "masterpiece, best quality, safe, 1girl, solo, upper body, looking at viewer, smile, garden, sunlight"
|
| 32 |
+
report = {"device": torch.cuda.get_device_name(), "cases": []}
|
| 33 |
+
original = {k: v.clone() for k, v in pipe.targets.items() if k in [
|
| 34 |
+
("transformer", "transformer_blocks.0.attn1.to_q.weight"),
|
| 35 |
+
next(k for k in pipe.targets if k[0] == "conditioner")]}
|
| 36 |
+
images = []
|
| 37 |
+
for key in [*MODELS, "base-v1.0"]:
|
| 38 |
+
model = MODELS[key]
|
| 39 |
+
filename = str(Path(a.checkpoints) / model.filename) if key != "base-v1.0" else None
|
| 40 |
+
start = time.perf_counter()
|
| 41 |
+
image = pipe.generate(key, filename, ref, prompt, steps=model.steps, cfg=model.cfg,
|
| 42 |
+
width=a.width, height=a.height, seed=0)
|
| 43 |
+
pixels = np.asarray(image)
|
| 44 |
+
assert pixels.std() > 5, (key, pixels.std())
|
| 45 |
+
suffix = "-restored" if len(report["cases"]) == len(MODELS) else ""
|
| 46 |
+
image.save(out / f"{key}{suffix}.png")
|
| 47 |
+
if suffix:
|
| 48 |
+
for k, value in original.items():
|
| 49 |
+
assert torch.equal(value, pipe.targets[k]), k
|
| 50 |
+
assert np.array_equal(np.asarray(images[0][1]), pixels), "Base image changed after round trip"
|
| 51 |
+
else:
|
| 52 |
+
images.append((model.label, image))
|
| 53 |
+
report["cases"].append({"model": key, "restored": bool(suffix), "steps": model.steps,
|
| 54 |
+
"cfg": model.cfg, "seconds": round(time.perf_counter()-start, 2),
|
| 55 |
+
"pixel_std": float(pixels.std())})
|
| 56 |
+
print(json.dumps(report["cases"][-1]), flush=True)
|
| 57 |
+
(out / "report.json").write_text(json.dumps(report, indent=2))
|
| 58 |
+
sheet = Image.new("RGB", (7*256, 420), "#eeeaf3")
|
| 59 |
+
draw = ImageDraw.Draw(sheet)
|
| 60 |
+
for i, (title, im) in enumerate([("Reference", ref), *images]):
|
| 61 |
+
sheet.paste(ImageOps.contain(im, (256, 384)), (256*i, 36))
|
| 62 |
+
draw.text((256*i+8, 12), title, fill="black")
|
| 63 |
+
sheet.save(out / "comparison.jpg")
|
| 64 |
+
print("PASS: six models generate; Base → variants → Base restores exact weights and pixels.")
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
if __name__ == "__main__":
|
| 68 |
+
main()
|
tests/test_space.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import tempfile
|
| 2 |
+
import unittest
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
from types import SimpleNamespace
|
| 5 |
+
from unittest.mock import patch
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
from safetensors.torch import save_file
|
| 9 |
+
|
| 10 |
+
from checkpoints import map_key, native_plan, copy_plan, diffusers_plan
|
| 11 |
+
from models import MODELS, model_settings
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class CheckpointTests(unittest.TestCase):
|
| 15 |
+
def test_switch_roundtrip_never_accumulates_and_includes_conditioner(self):
|
| 16 |
+
targets = {("transformer", "patch_embed.proj.weight"): torch.zeros(2, 3),
|
| 17 |
+
("conditioner", "proj.weight"): torch.zeros(3, 2)}
|
| 18 |
+
with tempfile.TemporaryDirectory() as tmp:
|
| 19 |
+
paths = []
|
| 20 |
+
for value in (1., 5.):
|
| 21 |
+
path = str(Path(tmp) / f"{value}.safetensors")
|
| 22 |
+
save_file({"diffusion_model.x_embedder.proj.1.weight": torch.full((2, 3), value),
|
| 23 |
+
"diffusion_model.llm_adapter.proj.weight": torch.full((3, 2), value + 1)}, path)
|
| 24 |
+
paths.append(path)
|
| 25 |
+
for i in (0, 1, 0, 1, 0):
|
| 26 |
+
copy_plan(native_plan(paths[i], targets), targets)
|
| 27 |
+
self.assertTrue(torch.equal(targets["transformer", "patch_embed.proj.weight"],
|
| 28 |
+
torch.full((2, 3), (1., 5.)[i])))
|
| 29 |
+
self.assertTrue(torch.equal(targets["conditioner", "proj.weight"],
|
| 30 |
+
torch.full((3, 2), (2., 6.)[i])))
|
| 31 |
+
|
| 32 |
+
def test_invalid_checkpoint_does_not_modify_any_weights(self):
|
| 33 |
+
targets = {("transformer", "patch_embed.proj.weight"): torch.ones(2, 3)}
|
| 34 |
+
cases = [{"unknown": torch.zeros(2, 3)}, {},
|
| 35 |
+
{"x_embedder.proj.1.weight": torch.zeros(3, 2)},
|
| 36 |
+
{"x_embedder.proj.1.weight": torch.zeros(2, 3, dtype=torch.int8)}]
|
| 37 |
+
with tempfile.TemporaryDirectory() as tmp:
|
| 38 |
+
path = str(Path(tmp) / "bad.safetensors")
|
| 39 |
+
for state in cases:
|
| 40 |
+
save_file(state, path)
|
| 41 |
+
with self.assertRaises(ValueError):
|
| 42 |
+
native_plan(path, targets)
|
| 43 |
+
self.assertTrue(torch.equal(next(iter(targets.values())), torch.ones(2, 3)))
|
| 44 |
+
|
| 45 |
+
def test_diffusers_restores_both_components(self):
|
| 46 |
+
targets = {("transformer", "weight"): torch.zeros(2),
|
| 47 |
+
("conditioner", "weight"): torch.zeros(2)}
|
| 48 |
+
with tempfile.TemporaryDirectory() as tmp:
|
| 49 |
+
for folder, value in [("transformer", 3.), ("text_conditioner", 7.)]:
|
| 50 |
+
root = Path(tmp) / folder
|
| 51 |
+
root.mkdir()
|
| 52 |
+
save_file({"weight": torch.full((2,), value)}, root / "diffusion_pytorch_model.safetensors")
|
| 53 |
+
copy_plan(diffusers_plan(tmp, targets), targets)
|
| 54 |
+
self.assertEqual(targets["transformer", "weight"].tolist(), [3., 3.])
|
| 55 |
+
self.assertEqual(targets["conditioner", "weight"].tolist(), [7., 7.])
|
| 56 |
+
|
| 57 |
+
def test_all_block_mapping_and_defaults(self):
|
| 58 |
+
for block in range(28):
|
| 59 |
+
self.assertEqual(map_key(f"blocks.{block}.self_attn.q_proj.weight"),
|
| 60 |
+
("transformer", f"transformer_blocks.{block}.attn1.to_q.weight"))
|
| 61 |
+
self.assertEqual(len(MODELS), 6)
|
| 62 |
+
for key in MODELS:
|
| 63 |
+
steps, cfg, _ = model_settings(key)
|
| 64 |
+
self.assertEqual((steps, cfg), (8, 1.) if key.startswith("turbo") else (28, 4.))
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
class SamplerTests(unittest.TestCase):
|
| 68 |
+
def test_cfg_one_skips_unconditional_but_preserves_reference(self):
|
| 69 |
+
from anima_role_ref_kv.pipeline import sample
|
| 70 |
+
positive, reference = torch.zeros(1), torch.zeros(1)
|
| 71 |
+
net = SimpleNamespace(dtype=torch.float32)
|
| 72 |
+
for cfg in (1., 4.):
|
| 73 |
+
calls, progress = [], []
|
| 74 |
+
def velocity(net, x, sigma, context, ref_kv=None):
|
| 75 |
+
calls.append((context, ref_kv))
|
| 76 |
+
return torch.ones_like(x)
|
| 77 |
+
with patch("anima_role_ref_kv.pipeline.reference_ops.encode_reference", return_value="kv") as encode, \
|
| 78 |
+
patch("anima_role_ref_kv.pipeline.reference_ops.velocity", side_effect=velocity):
|
| 79 |
+
output = sample(net, positive, "negative", reference, width=16, height=16,
|
| 80 |
+
steps=3, cfg=cfg, progress_callback=lambda i, n: progress.append((i, n)))
|
| 81 |
+
encode.assert_called_once()
|
| 82 |
+
self.assertEqual(len(calls), 3 if cfg == 1 else 6)
|
| 83 |
+
self.assertEqual(sum(kv == "kv" for _, kv in calls), 3)
|
| 84 |
+
self.assertEqual(progress, [(1, 3), (2, 3), (3, 3)])
|
| 85 |
+
expected = torch.randn(1, 16, 1, 2, 2, generator=torch.Generator().manual_seed(0)) - 1
|
| 86 |
+
torch.testing.assert_close(output, expected)
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
if __name__ == "__main__":
|
| 90 |
+
unittest.main()
|