Solve[{(197*x1)/125 + (24*y1)/125 - ((18*x3)/25 + (6*y3)/25)^2/((9*x6)/10 + (3*y6)/10 + 3) - 3/25 == 0,
(27*x1)/25 + (43*x2)/25 - (((27*x3)/20 + (9*x5)/10)*((18*x3)/25 + (6*y3)/25))/((9*x6)/ 10 + (3*y6)/10 + 3) == 0,
x3 + (9*y1)/25 + (6*y2)/ 25 - (((18*x3)/25 + (6*y3)/25)*((9*y3)/20 + (3*y5)/10))/((9*x6)/10 + (3*y6)/10 + 3) == 0,
(81*x1)/40 + (27*x2)/10 + (19*x4)/10 - ((27*x3)/20 + (9*x5)/10)^2/((9*x6)/10 + (3*y6)/10 + 3) - 9 == 0,
x5 - (((27*x3)/20 + (9*x5)/10)*((9*y3)/20 + (3*y5)/10))/((9*x6)/10 + (3*y6)/10 + 3) == 0,
x6 + (27*y1)/40 + (9*y2)/10 + (3*y4)/10 - ((9*y3)/20 + (3*y5)/10)^2/((9*x6)/10 + (3*y6)/10 + 3) == 0,
(8*x1)/125 + (181*y1)/125 - ((2*x3)/25 + (14*y3)/25)^2/(x6/10 + (7*y6)/10 + 7) - 2/25 ==0,
(3*x1)/25 + (2*x2)/25 + y2 - (((3*x3)/20 + x5/10)*((2*x3)/25 + (14*y3)/25))/(x6/10 + (7*y6)/10 + 7) == 0,
(21*y1)/25 + (14*y2)/25 + y3 - (((2*x3)/25 + (14*y3)/25)*((21*y3)/20 + (7*y5)/10))/(x6/ 10 + (7*y6)/10 + 7) == 0,
(9*x1)/40 + (3*x2)/10 + x4/10 +y4 - ((3*x3)/20 + x5/10)^2/(x6/10 + (7*y6)/10 + 7) - 1 == 0,
y5 - (((3*x3)/20 + x5/10)*((21*y3)/20 + (7*y5)/10))/(x6/10 + (7*y6)/10 + 7) == 0,
(63*y1)/40 + (21*y2)/10 + (7*y4)/10 + y6 - ((21*y3)/20 + (7*y5)/10)^2/(x6/10 + (7*y6)/10 + 7) == 0 }, {x1, x2, x3, x4, x5, x6, y1, y2, y3, y4, y5, y6}]

返回空列表说明无解。
此外你这是二次方程组。

本人试过，先用FullSimplify化解，再用NSolve求解，可以解出数值解，有实数解有复数解，{x4 -> 11.1491 - 3.29227 I, y4 -> 0.204157 + 0.572793 I,
x2 -> -0.0781485 + 0.16914 I, x6 -> -0.304697 + 0.496725 I,
y6 -> -9.74473 + 0.127651 I, y2 -> -0.00385791 - 0.0257475 I,
x1 -> 0.0671563 - 0.00317183 I, y1 -> 0.0521563 + 0.000372254 I,
x5 -> 2.27028 + 1.99241 I, y5 -> -0.699308 + 1.8945 I,
x3 -> -0.0827949 + 0.0186306 I,
y3 -> 0.00927444 - 0.0181662 I},
{x4 -> 4.84295, y4 -> 1.41751,
x2 -> -0.13534, x6 -> -4.67942, y6 -> -9.15046, y2 -> 1.98082,
x1 -> -0.793711, y1 -> 3.64618, x5 -> -0.134815, y5 -> 1.09117,
x3 -> -1.87099, y3 -> -1.17523}, {x4 -> -18.9353, y4 -> 4.60217,
x2 -> -4.92117, x6 -> -2.04862, y6 -> -4.45192, y2 -> 0.522878,
x1 -> -0.871856, y1 -> 0.105316, x5 -> -2.75335, y5 -> 0.0347973,
x3 -> -0.594826, y3 -> -0.357619},